{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.base import BaseEstimator, ClassifierMixin\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from sklearn.utils import check_X_y\n",
    "from sklearn.utils import check_random_state\n",
    "from sklearn.utils import shuffle\n",
    "import theano\n",
    "import theano.tensor as tt\n",
    "from theano import shared\n",
    "from time import time\n",
    "from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset...\n",
      "Creating train-test split...\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_mldata\n",
    "from sklearn.utils import check_array\n",
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "\n",
    "def load_data(dtype=np.float32, order='C'):\n",
    "    \"\"\"Load the data, then cache and memmap the train/test split\"\"\"\n",
    "    print(\"Loading dataset...\")\n",
    "    data = fetch_mldata('MNIST original')\n",
    "    X = check_array(data['data'], dtype=dtype, order=order)\n",
    "    y = data[\"target\"].astype(np.int32)\n",
    "    # Scale the features to [0, 1]\n",
    "    X = X / 255\n",
    "    ## Create train-test split (as [Joachims, 2006])\n",
    "    print(\"Creating train-test split...\")\n",
    "    n_train = 60000\n",
    "    X_train = X[:n_train]\n",
    "    y_train = y[:n_train]\n",
    "    X_test = X[n_train:]\n",
    "    y_test = y[n_train:]\n",
    "    return X_train, X_test, y_train, y_test\n",
    "\n",
    "X_dev, X_test, y_dev, y_test = load_data()\n",
    "\n",
    "# Create a small training set for faster experiments\n",
    "X_train, X_val, y_train, y_val = train_test_split(\n",
    "    X_dev, y_dev, test_size=int(1e3), random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Various neural networks utilities\n",
    "\n",
    "Mostly taken from Alec Radford and Kyle Kastner's gists:\n",
    "\n",
    "- https://gist.github.com/Newmu/a56d5446416f5ad2bbac\n",
    "- https://gist.github.com/kastnerkyle/f3f67424adda343fef40"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def sharedf32(X, dtype=np.float32, name=None):\n",
    "    return theano.shared(np.asarray(X, dtype=dtype), name=name)\n",
    "\n",
    "\n",
    "def softmax(x):\n",
    "    return tt.nnet.softmax(x)\n",
    "\n",
    "\n",
    "def relu(x):\n",
    "    return (x + abs(x)) / 2.0\n",
    "\n",
    "\n",
    "def tanh(x):\n",
    "    return tt.tanh(x)\n",
    "\n",
    "\n",
    "def sigmoid(x):\n",
    "    return tt.nnet.sigmoid(x)\n",
    "\n",
    "\n",
    "def linear(x):\n",
    "    return x\n",
    "\n",
    "\n",
    "def iter_data(*data, **kwargs):\n",
    "    batch_size = kwargs.get('batch_size', 128)\n",
    "    batches, remainder = divmod(len(data[0]), batch_size)\n",
    "    if remainder != 0:\n",
    "        batches += 1\n",
    "    for b in range(batches):\n",
    "        start = b * batch_size\n",
    "        end = (b + 1) * batch_size\n",
    "        if len(data) == 1:\n",
    "            yield data[0][start:end]\n",
    "        else:\n",
    "            yield tuple([d[start:end] for d in data])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimizers "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# XXX: do not use this. it probably has a bug\n",
    "\n",
    "class Adam(object):\n",
    "    \"\"\"no bias init correction and no b1 decay\"\"\"\n",
    "    \n",
    "    def __init__(self, learning_rate=0.01, grad_momentum=0.1,\n",
    "                 sq_grad_momentum=0.001, eps=1e-8):\n",
    "        self.learning_rate = learning_rate\n",
    "        self.grad_momentum = grad_momentum\n",
    "        self.sq_grad_momentum = sq_grad_momentum\n",
    "        self.eps = eps\n",
    "\n",
    "    def make_updates(self, params, cost):\n",
    "        b1 = np.array(self.grad_momentum, dtype=np.float32)\n",
    "        b2 = np.array(self.sq_grad_momentum, dtype=np.float32)\n",
    "        lr = np.array(self.learning_rate, dtype=np.float32)\n",
    "        eps = np.array(self.eps, dtype=np.float32)\n",
    "        one = np.array(1, dtype=np.float32)\n",
    "        updates = []\n",
    "        grads = tt.grad(cost, params)\n",
    "        for p, g in zip(params, grads):\n",
    "            m = sharedf32(p.get_value() * 0., name='m')\n",
    "            v = sharedf32(p.get_value() * 0., name='v')\n",
    "            m_t = (b1 * g) + ((one - b1) * m)\n",
    "            v_t = (b2 * tt.sqr(g)) + ((one - b2) * v)\n",
    "            g_t = m_t / (tt.sqrt(v_t) + eps)\n",
    "            p_t = p - lr * g_t\n",
    "            updates.append((m, m_t))\n",
    "            updates.append((v, v_t))\n",
    "            updates.append((p, p_t))\n",
    "        return updates\n",
    "    \n",
    "\n",
    "class MomentumSGD(object):\n",
    "    \n",
    "    monitor = None\n",
    "    \n",
    "    def __init__(self, learning_rate_init=0.1, max_adaptation_count=3,\n",
    "                 learning_rate_scale=0.2, momentum=0.9, rescale=0.5):\n",
    "        self.learning_rate_init = learning_rate_init\n",
    "        self.max_adaptation_count = max_adaptation_count\n",
    "        self.learning_rate_scale = learning_rate_scale\n",
    "        self.momentum = momentum\n",
    "        self.rescale = rescale\n",
    "        \n",
    "    def on_stalled_convergence(self):\n",
    "        if self.adaptation_count > self.max_adaptation_count:\n",
    "            # Optimizer thinks model has converged\n",
    "            return False\n",
    "        \n",
    "        scale = np.array(self.learning_rate_scale, dtype=np.float32)\n",
    "        \n",
    "        # Rescale the learning rate\n",
    "        old_lr = self.learning_rate.get_value()\n",
    "        new_lr = old_lr * scale\n",
    "        self.learning_rate.set_value(new_lr)\n",
    "        self.adaptation_count += 1\n",
    "        print(\"Updated learning rate from %f to %f\" % (old_lr, new_lr))\n",
    "\n",
    "    def make_updates(self, params, cost):\n",
    "        self.adaptation_count = 0\n",
    "        self.learning_rate = sharedf32(self.learning_rate_init,\n",
    "                                       name='learning_rate')\n",
    "        momentum = np.array(self.momentum, dtype=np.float32)\n",
    "\n",
    "        updates = []\n",
    "        grads = tt.grad(cost, params)\n",
    "        for p, g in zip(params, grads):\n",
    "            if self.momentum > 0:\n",
    "                v = sharedf32(p.get_value() * 0., name='v')\n",
    "                v_t = momentum * v - self.learning_rate * g\n",
    "                p_t = p + v_t\n",
    "                updates.append((v, v_t))\n",
    "            else:\n",
    "                # traditional SGD\n",
    "                p_t = p - self.learning_rate * g\n",
    "            updates.append((p, p_t))\n",
    "        return updates\n",
    "\n",
    "    \n",
    "class NesterovMomentumSGD(MomentumSGD):\n",
    "\n",
    "    def make_updates(self, params, cost):\n",
    "        self.adaptation_count = 0\n",
    "        self.learning_rate = sharedf32(self.learning_rate_init,\n",
    "                                       name='learning_rate')\n",
    "        momentum = np.array(self.momentum, dtype=np.float32)\n",
    "        updates = []\n",
    "        grads = tt.grad(cost, params)\n",
    "#         grad_norm = tt.sqrt(sum(map(lambda x: tt.sqr(x).sum(), grads)))\n",
    "#         param_norm = tt.sqrt(sum(map(lambda x: tt.sqr(x).sum(), params)))\n",
    "#         rescale = np.array(self.rescale, dtype=np.float32)\n",
    "#         not_finite = tt.or_(tt.isnan(grad_norm), tt.isinf(grad_norm))\n",
    "#         scaling_num = rescale\n",
    "#         scaling_den = tt.maximum(rescale, grad_norm)\n",
    "    \n",
    "        tenth = np.array(0.1, dtype=np.float32)\n",
    "        for p, g in zip(params, grads):\n",
    "#             g = tt.switch(not_finite, tenth * p, g * (scaling_num / scaling_den))\n",
    "            v = sharedf32(p.get_value() * 0., name='v')\n",
    "            v_t = momentum * v - self.learning_rate * g\n",
    "            p_t = p + momentum * v_t - self.learning_rate * g\n",
    "            updates.append((v, v_t))\n",
    "            updates.append((p, p_t))\n",
    "        return updates\n",
    "#         return grad_norm, param_norm, updates"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Orchestration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class EarlyStoppingMonitor(object):\n",
    "    def __init__(self, X_train, y_train, X_val=None, y_val=None, patience=5,\n",
    "                 data_log_filename=None, subsample=int(5e3), tol=1e-5, random_state=None):\n",
    "\n",
    "        X_train, y_train = check_X_y(X_train, y_train, dtype=np.float32)\n",
    "        y_train = y_train.astype(np.int32, copy=False)\n",
    "        self.label_binarizer_ = lb = LabelBinarizer().fit(y_train)\n",
    "\n",
    "        if subsample is not None and subsample < X_train.shape[0]:\n",
    "            X_train, y_train = shuffle(X_train, y_train, random_state=random_state)\n",
    "            X_train = X_train[:subsample]\n",
    "            y_train = y_train[:subsample]\n",
    "        self.X_train = X_train\n",
    "        self.y_train = y_train\n",
    "        self.Y_train = lb.transform(y_train).astype(np.float32)\n",
    "        self.tol = tol\n",
    "\n",
    "        if X_val is not None:\n",
    "            X_val, y_val = check_X_y(X_val, y_val, dtype=np.float32)\n",
    "            if subsample is not None and subsample < X_val.shape[0]:\n",
    "                X_val, y_val = shuffle(X_val, y_val,\n",
    "                                       random_state=random_state)\n",
    "                X_val = X_val[:subsample]\n",
    "                y_val = y_val[:subsample]\n",
    "            self.Y_val = lb.transform(y_val).astype(np.float32)\n",
    "        self.X_val = X_val\n",
    "        self.y_val = y_val\n",
    "\n",
    "        self.data_log_filename = data_log_filename\n",
    "        self.patience_reset = patience\n",
    "        self.reset()\n",
    "    \n",
    "    def reset(self):\n",
    "        self.best_cost = np.inf\n",
    "        self.patience = self.patience_reset\n",
    "        self.timestamp = time()\n",
    "    \n",
    "    def accuracy(self, y_true, output):\n",
    "        indices = output.argmax(axis=1)\n",
    "        return np.mean(self.label_binarizer_.classes_[indices] == y_true)\n",
    "        \n",
    "    def monitor(self, model, context):\n",
    "        duration = time() - self.timestamp\n",
    "        epoch = context.get('epoch')\n",
    "        optimizer = context.get('optimizer')\n",
    "        \n",
    "        training_out, training_cost = model._forward_and_cost(\n",
    "            self.X_train, self.y_train)\n",
    "        training_acc = self.accuracy(self.y_train, training_out)\n",
    "        if self.X_val is not None:\n",
    "            validation_out, validation_cost = model._forward_and_cost(\n",
    "                self.X_val, self.y_val)\n",
    "            validation_acc =  self.accuracy(self.y_val, validation_out)\n",
    "            current_cost = validation_cost      \n",
    "        else:\n",
    "            validation_cost = None\n",
    "            validation_acc = None\n",
    "            current_cost = training_cost\n",
    "\n",
    "        msg = \"Epoch #%03d, training cost: %0.6f, acc: %0.4f\" % (\n",
    "            epoch, training_cost, training_acc)\n",
    "        \n",
    "        if self.X_val is not None:\n",
    "            msg += \", validation cost: %0.6f, acc: %0.4f\" % (\n",
    "                validation_cost, validation_acc)\n",
    "        msg += \", duration: %0.3fs \" % duration\n",
    "        print(msg)\n",
    "\n",
    "        # Patience-based stopping condition\n",
    "        if current_cost < self.best_cost - self.tol:\n",
    "            self.patience = self.patience_reset\n",
    "            self.best_cost = current_cost\n",
    "        elif (hasattr(optimizer, 'on_stalled_convergence')\n",
    "              and not optimizer.on_stalled_convergence()):\n",
    "            # Optimizer has already had enough opportunities to adapt the\n",
    "            # learning rate in the past. We can start loosing patience.\n",
    "            self.patience -= 1\n",
    "        else:\n",
    "            # The optimizer does not use validation set convergence info,\n",
    "            # we can loose patience without notifying it.\n",
    "            self.patience -= 1\n",
    "\n",
    "        self.timestamp = time()\n",
    "        return self.patience > 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def shuffle_columns(X, copy=True, random_state=None):\n",
    "    rng = check_random_state(random_state)\n",
    "    if copy:\n",
    "        X = X.copy()\n",
    "    for i in range(X.shape[1]):\n",
    "        rng.shuffle(X[:, i])\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class MLPClassifier(BaseEstimator, ClassifierMixin):\n",
    "    \n",
    "    def __init__(self, hidden=(100, 100), activation='relu', init_gain='auto',\n",
    "                 batch_size=128, optimizer=None,\n",
    "                 weight_decay=int(1e-5), warm_start=False,\n",
    "                 contrast_ratio=.5,\n",
    "                 max_iter=1000, random_state=None):\n",
    "        self.hidden = hidden\n",
    "        self.activation = activation\n",
    "        self.batch_size = batch_size\n",
    "        self.optimizer = optimizer\n",
    "        self.weight_decay = weight_decay\n",
    "        self.init_gain = init_gain\n",
    "        self.warm_start = warm_start\n",
    "        self.random_state = random_state\n",
    "        self.max_iter = max_iter\n",
    "        self.contrast_ratio = contrast_ratio\n",
    "        \n",
    "    def _init_parameters(self, n_features, n_outputs):\n",
    "        rng = check_random_state(self.random_state)\n",
    "        input_dims = (n_features,) + self.hidden\n",
    "        output_dims = self.hidden + (n_outputs,)\n",
    "\n",
    "        if self.init_gain == 'auto':\n",
    "            g = np.sqrt(2) if self.activation == 'relu' else 1.\n",
    "        else:\n",
    "            g = self.init_gain\n",
    "        \n",
    "        self.weights_ = []\n",
    "        self.biases_ = []\n",
    "        for l, (in_dim, out_dim) in enumerate(zip(input_dims, output_dims)):\n",
    "            std = g * np.sqrt(2 / (in_dim + out_dim))\n",
    "            W = sharedf32(rng.normal(size=(in_dim, out_dim), scale=std),\n",
    "                          name='W_%d' % l)\n",
    "            self.weights_.append(W)\n",
    "            b = sharedf32(np.zeros(out_dim), name='b_%d' % l)\n",
    "            self.biases_.append(b)\n",
    "        \n",
    "    def _make_functions(self, contrast_ratio):\n",
    "        x = tt.fmatrix(name='x')\n",
    "        y = tt.ivector(name='y')\n",
    "        contrast_activations = tt.fmatrix(name='contrast_activations')\n",
    "\n",
    "        # Define the computation graph of the model\n",
    "        if self.activation == 'relu':\n",
    "            sigma = relu\n",
    "        elif self.activation == 'tanh':\n",
    "            sigma = tanh\n",
    "        elif self.activation == 'linear':\n",
    "            sigma = linear\n",
    "        else:\n",
    "            raise ValueError('Unsupported activation: %s' % self.activation)\n",
    "        \n",
    "        transfer_funcs = [sigma] * (len(self.weights_) - 1) + [softmax]\n",
    "        n_layers = len(transfer_funcs)\n",
    "        activations = []\n",
    "        activation = x\n",
    "        for l, w, b, s in zip(range(n_layers), self.weights_,\n",
    "                              self.biases_, transfer_funcs):\n",
    "            activation = s(tt.dot(activation, w) + b)\n",
    "            activation.name = 'activations_%d' % l\n",
    "            activations.append(activation)\n",
    "        \n",
    "        output = activation\n",
    "        classification_cost = tt.nnet.categorical_crossentropy(output, y).mean()\n",
    "        contrast_cost = (tt.dot(activations[-2], contrast_activations.T) ** 2).mean()\n",
    "        \n",
    "        if contrast_ratio > 0 and contrast_ratio < 1:\n",
    "            cost = (1 - self.contrast_ratio) * classification_cost\n",
    "            cost += self.contrast_ratio * contrast_cost\n",
    "        elif contrast_ratio == 1:\n",
    "            cost = contrast_cost\n",
    "        else:\n",
    "            cost = classification_cost\n",
    "        \n",
    "        weight_decay = np.array(self.weight_decay, dtype=np.float32)\n",
    "        if self.weight_decay > 0:\n",
    "            for w in self.weights_:\n",
    "                cost += weight_decay * tt.sqr(w).sum()\n",
    "        \n",
    "        # Use the optimizer to compute the parameter updates based\n",
    "        # on the gradient of the cost function\n",
    "        opt = self.optimizer\n",
    "        if opt is None:\n",
    "            opt = Adam()\n",
    "\n",
    "        parameters = []\n",
    "        parameters += self.weights_\n",
    "        parameters += self.biases_\n",
    "#         grad_norm, param_norm, fit_updates = opt.make_updates(parameters, cost)\n",
    "        fit_updates = opt.make_updates(parameters, cost)\n",
    "        \n",
    "        # Compile the functions them-selves\n",
    "        f = theano.function\n",
    "        if contrast_ratio > 0 and contrast_ratio < 1:\n",
    "            self._fit = f([x, y, contrast_activations],\n",
    "              cost, updates=fit_updates, name='_fit')\n",
    "        elif contrast_ratio == 0:\n",
    "            self._fit = f([x, y], cost, updates=fit_updates, name='_fit')\n",
    "        else:\n",
    "            self._fit = f([x, contrast_activations],\n",
    "                          cost, updates=fit_updates, name='_fit')\n",
    "\n",
    "#         self._fit = f([x, y], (cost, grad_norm, param_norm),\n",
    "#                       updates=fit_updates, name='_fit')\n",
    "        self._forward_and_cost = f([x, y], (output, classification_cost), name=\"_forward_and_cost\")\n",
    "        self._forward = f([x], output, name=\"_forward\")\n",
    "        inputs = [x] + activations[:-1]\n",
    "        self._forward_funcs = []\n",
    "        for i, input_ in enumerate(inputs):\n",
    "            self._forward_funcs.append(f([input_], activations[i:],\n",
    "                                         name=\"_forward_activations_%d\" % i))\n",
    "        return opt\n",
    "    \n",
    "    def forward_shuffle(self, X, layer_idx=0, random_state=None):\n",
    "        if random_state is None:\n",
    "            random_state = self.random_state\n",
    "        if layer_idx == -1:\n",
    "            all_activations = []\n",
    "            activations = X.copy()\n",
    "        else:\n",
    "            all_activations = self._forward_funcs[0](X)\n",
    "            activations = all_activations[layer_idx]\n",
    "        shuffle_activations = shuffle_columns(activations, random_state=random_state)\n",
    "        after_shuffle_activations = self._forward_funcs[layer_idx + 1](shuffle_activations)\n",
    "        return all_activations[:layer_idx + 1] + [shuffle_activations] + after_shuffle_activations\n",
    "\n",
    "    def fit(self, X, y=None, monitor=None):\n",
    "        X, y = check_X_y(X, y, dtype=np.float32)\n",
    "        y = y.astype(np.int32, copy=False)\n",
    "        self.label_binarizer_ = lb = LabelBinarizer()\n",
    "        Y = lb.fit_transform(y).astype(np.float32)\n",
    "        n_samples, n_features = X.shape\n",
    "        _, n_outputs = Y.shape\n",
    "        if not self.warm_start or not hasattr(self, 'weights_') :\n",
    "            self._init_parameters(n_features, n_outputs)\n",
    "\n",
    "        contrast_ratio = self.contrast_ratio if y is not None else 1.\n",
    "        optimizer = self._make_functions(contrast_ratio)\n",
    "        self.training_costs_ = []\n",
    "        if monitor is not None:\n",
    "            monitor.reset()\n",
    "\n",
    "        try:\n",
    "            for epoch in range(self.max_iter):\n",
    "                if monitor is not None and not monitor.monitor(self, locals()):\n",
    "                    break\n",
    "                for X_batch, y_batch in iter_data(X, y, batch_size=self.batch_size):\n",
    "#                     cost, grad_norm, param_norm = self._fit(X_batch, y_batch)\n",
    "                    if self.contrast_ratio > 0:\n",
    "                        layer_idx = np.random.randint(-1, 2)\n",
    "                        contrast_activations = self.forward_shuffle(X_batch, layer_idx=layer_idx)[-2]\n",
    "                        cost = self._fit(X_batch, y_batch, contrast_activations)\n",
    "                    else:\n",
    "                        cost = self._fit(X_batch, y_batch)\n",
    "#                     print(grad_norm, param_norm)\n",
    "                    self.training_costs_.append(cost)\n",
    "        except KeyboardInterrupt:\n",
    "            print(\"Interruped by user\")\n",
    "\n",
    "        return self\n",
    "    \n",
    "    def predict_proba(self, X):\n",
    "        X = check_array(X, dtype=np.float32)\n",
    "        return self._forward(X)\n",
    "\n",
    "    def predict(self, X):\n",
    "        indices = self.predict_proba(X).argmax(axis=1)\n",
    "        return indices\n",
    "#         return self.label_binarizer_.classes_[indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch #000, training cost: 2.469210, acc: 0.0854, validation cost: 2.489720, acc: 0.0780, duration: 0.000s \n",
      "Epoch #001, training cost: 0.111276, acc: 0.9660, validation cost: 0.142365, acc: 0.9580, duration: 26.942s \n",
      "Epoch #002, training cost: 0.071616, acc: 0.9804, validation cost: 0.091475, acc: 0.9730, duration: 26.623s \n",
      "Epoch #003, training cost: 0.048391, acc: 0.9856, validation cost: 0.074663, acc: 0.9790, duration: 26.828s \n",
      "Interruped by user\n",
      "CPU times: user 3min 36s, sys: 4.89 s, total: 3min 41s\n",
      "Wall time: 1min 48s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "monitor = EarlyStoppingMonitor(X_train, y_train, X_val=X_val, y_val=y_val,\n",
    "                               patience=10, random_state=0)\n",
    "train_only_monitor = EarlyStoppingMonitor(X_train, y_train, random_state=0)\n",
    "\n",
    "optimizer = NesterovMomentumSGD(learning_rate_init=.1, learning_rate_scale=0.5, momentum=0.9)\n",
    "mlp = MLPClassifier(hidden=(1000, 1000), batch_size=128, weight_decay=1e-4,\n",
    "                    contrast_ratio=0.005,\n",
    "                    init_gain='auto', optimizer=optimizer, max_iter=100, random_state=0)\n",
    "mlp.fit(X_train, y_train, monitor=monitor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9753 0.0247\n"
     ]
    }
   ],
   "source": [
    "test_score = mlp.score(X_test, y_test)\n",
    "print(test_score, 1 - test_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPwAAAD8CAYAAABTq8lnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGB9JREFUeJzt3Xts1GW6B/DvA3IvguVSSi2Uu1BAClqpwoIKBkVdTQwr\nhsSsl+wf63qBxMv+cdyck3jWNWvM+cNNjuKG3bPKrkU4oMAREBUvXESgUMq9hVKg5SoiigWe80en\ndSgzz1OmnemY9/tJjNN+Z6a//joPc3nf93lFVUFEYWjT2gdARKnDgicKCAueKCAseKKAsOCJAsKC\nJwpIwgUvItNEZIeI7BaR51ryoIgoOSSRcXgRaQtgJ4ApAKoAbAAwU1XLoq7DAX6iVqSq0vh7VyV4\nX4UA9qhqBQCIyHwAvwRQFn2lJ598suHy2rVrMX78+Iav+/XrZ/6AoqIiM1+wYIGZZ2dnm3nHjh0v\n+Xrp0qW46667Gr7eunWrefupU6ea+fbt2838u+++M/P8/PxLvl64cCHuv//+hq/PnTtn3v7IkSNm\nPmrUKDMXueyxcpni4uKGy1u2bMH1119/ST579mzz9uXl5WZ+4sQJM9+7d6+Z5+TkNFxu/PcFgCFD\nhpi3//HHH828qqrKzK+++mozj/79Yh3fxYsXzdt///33cbPnn38+5vcTfUmfA6Ay6uuDke8RURpL\ntOD5cp3oZyjRl/RVAHKjvs5F3bP8JdauXdtwuUOHDgn+qNTwXt61tuuuu661D8GUlZXV2odgSve/\nb3OPb+/evdi3b597vUQL/isAQ0QkD8AhAL8CMLPxlaLfs6e7dH9ADB8+vLUPwdSnT5/WPgRTuv99\nm3t8gwYNwqBBgxq+XrVqVczrJVTwqnpeRJ4A8H8A2gKYG/0JPRGlp4SG5Zp0xyK6bNmyuPm6devM\n22dmZpp59L9msezZs8fMx4wZY+beJ7RlZfa/bz179jTzvn37mvnBg5e9Q7pERkaGmefm5pr5mTNn\nzLy2ttbMAaBdu3Zm3qaN/RFR9Fu+WLzHQGVlpZl7f2Pv/nfs2GHm3jnyRkq6detm5r169TJz6/w/\n9NBDMYflONOOKCAseKKAsOCJAsKCJwoIC54oICx4ooCw4IkCkuhMuyaxxsLbtm1r3rZHjx5mvmLF\nCjOfPn26mS9ZssTMvZlPFy5cMHNvNeA333xj5lddZf9pDh06ZObeGHFzx4AB4NixY2Z++vRpM2+8\nYrGxm2++2cw///xzM2/fvr2ZL1261Mx79+7drPv3zrE3T8Gb51BRUWHmMX/mFd+CiH62WPBEAWHB\nEwWEBU8UEBY8UUBY8EQBYcETBSSp4/DWOGWnTp3M23rj3NEdcWPx1lp37tzZzD3eOLXXtdYbp/e6\n7ubl5Zn5okWLzHzEiBFmPm/ePDMHgBtuuMHMBw8ebObN7Wng/Q5Hjx4185EjR5q517Nh7NixZu61\nJfN6LmzYsMHMvbkssfAZniggLHiigLDgiQLCgicKCAueKCAseKKAsOCJApLUcXhrnLW6utq87YED\nB8x84cKFZu7thDJ69Ggz99Yae+vVvTFkby24N0/g+PHjZu6NkXu7w3o7nwLA/v37zdzrfX/TTTeZ\nubfbTnPHqb2eBNburIC/N4C3w29NTY2Ze9uzde3a1cxj4TM8UUBY8EQBYcETBYQFTxQQFjxRQFjw\nRAFhwRMFpFnj8CJSAeA0gAsAalW1MDovLS2Ne1tvrbS3t7Y3xnnttdeauTcG6/UUz8nJMfMuXbqY\n+Zdffmnm3nr7oqIiM1+1apWZd+/e3czffvttMweA5557zsy9c1hWVmbm3jizl3/77bdm/sYbb5j5\n7NmzzdybB+D1tfd6HowbN87MT548aeaxNHfijQKYrKonmnk/RJQCLfGS3p6yRURpo7kFrwBWishX\nIvJ4SxwQESVPc1/S36Kqh0WkF4AVIrJDVdfUh8uWLWu44uDBg9392ogoMaWlpeZnZvWaVfCqejjy\n/6MishBAIYCGgr/zzjubc/dE1ET5+fnIz89v+Lq4uDjm9RJ+SS8inUWka+RyFwB3ANia6P0RUfI1\n5xk+C8DCyDLLqwD8Q1U/bJGjIqKkSLjgVbUcwBjrOhcvXoybeePM3vsRb4xz8eLFZu71FL/xxhvN\n/LvvvjNzb392b57AxIkTzXznzp1m7q1F98bpX3rpJTMH4H4m4+0f762H9/q2e+vVvXP8+OP258yn\nTp0yc6/v/fnz583cO38//PCDmZ89e9bMY+FMO6KAsOCJAsKCJwoIC54oICx4ooCw4IkCwoInCkhS\n+9Jbe6Bv3rzZvK3Xt94bY/X2DvfWwx8+fNjMvb3Nvb763jh9VlaWmQ8YMMDMvTFcr+d5ZmammQP+\nOfB6HnhzLby/sdd735sr4a2nP336tJlfc801Zr5r1y4zP3TokJl7f6Nbb73VzGPhMzxRQFjwRAFh\nwRMFhAVPFBAWPFFAWPBEAWHBEwUkqePw1jimN8bp9RT3xrG9/de9tdTeentrjgHgr4X2xtHz8vLM\nfN++fWburWefNWuWmU+aNMnMAb8vu5ePHj3azL25EN56dW9/eG+ewLp168zc27/e68s/dOhQM+/R\no4eZezUUC5/hiQLCgicKCAueKCAseKKAsOCJAsKCJwoIC54oIEkdh7f2IB84cKB521/84hdm7q1n\n99ZCT5gwwcy9cXjv/r2e5ZENPOI6fvy4ma9fv97Mp0+fbubnzp0z84MHD5o5AIwfP97Mvd74Xt/5\nyspKM/fmOniPMe93vOOOO8x8z549Zu6d45ycHDP3HuPt27c381j4DE8UEBY8UUBY8EQBYcETBYQF\nTxQQFjxRQFjwRAFxx+FF5C0A0wHUqOqoyPcyAfwTQH8AFQBmqOpli5MrKiri3q/X07tTp05mvnr1\najMvKioy86qqKjPfv3+/mXtjoP3792/Wz/d6tnu//7Zt28x82rRpZu6txweAjRs3mvlHH31k5t45\nLCwsNHNvroK3nv2qq+yHv9dzoHPnzmY+atQoM/f2DvBy7zESS1Oe4f8KoPGj43kAK1R1KIBVka+J\nKM25Ba+qawCcbPTtewHMi1yeB+C+Fj4uIkqCRN/DZ6lq/V5Q1QDsfZGIKC00ey69qqqIaKxs+fLl\nDZcHDx7s9hAjosSUl5ebn5nVS7Tgq0Wkj6oeEZFsADWxruR9MERELWPAgAGXNEb95JNPYl4v0Zf0\niwE8HLn8MIBFCd4PEaWQW/Ai8g6ALwAME5FKEfk1gD8CmCoiuwDcFvmaiNKc+5JeVWfGiaZ4t7V6\nr3/55Zfmbb29rwsKCszcW4/u9fz25gF4ffN37Nhh5hcvXjRzryd7SUmJmXvzELy+817ff8A/R97+\n5yNGjDDzjIwMMx8yZIiZe58ZnTzZePDpUrm5uWb+6aefmrnXF79du3Zm7p0/9qUnIhMLniggLHii\ngLDgiQLCgicKCAueKCAseKKAJLUvfZs28f898fbG9vquWz3vAX//9Q4dOpi5Ny/52LFjZu7tPz92\n7Fgzb+56/OLiYjOfMWOGmWdmZpo54I9TX3fddWb+2GOPmbk3V+LAgQNm7vW99/q+e7f35gF4exts\n3brVzL2+/9Y4/quvvhrz+3yGJwoIC54oICx4ooCw4IkCwoInCggLniggLHiigCR1HH737t1xM6+n\ntzeG6enTp4+Ze/u7e3uHL1682My9nuTeGO57771n5l5P9ldeecXM+/bta+beWmwAOHXqsq0ILvHM\nM8+Yubc3wZEjR8x8/vz5Zj5r1iwz93oaeHsHZGXZvVu9ng/WPBUA2Llzp5l78yBi/swrvgUR/Wyx\n4IkCwoInCggLniggLHiigLDgiQLCgicKSFLH4a09xr315F7Pc28ttDdO/e6775r52bNnzXzy5Mlm\n7u09/vTTT5u5Nw/Bm0fw+uuvm7nXFz8/P9/MAaCystLMvfXsa9asMfPs7Gwzv/vuu83c6xng5d56\n/u3bt5u59xgSETP39mbw1uvHwmd4ooCw4IkCwoInCggLniggLHiigLDgiQLCgicKiDsOLyJvAZgO\noEZVR0W+9wcAjwGo34T9BVVd3vi2W7ZsiXu/3t7ZXu6tN9+0aZOZ9+rVy8zPnz9v5l7feO/2Xl/8\n++67z8z37dtn5uXl5WY+bNgwM/eOH/DH4b2/gdcX3tvD3ltPf/ToUTP/+OOPzXzkyJFm7v0NvLkO\nXs8G7/iGDx9u5rE05Rn+rwCmNfqeAnhVVQsi/11W7ESUftyCV9U1AE7GiOxpQkSUdprzHv53IrJF\nROaKiL3vExGlhUTn0v8FwL9HLv8HgD8DeLTxlaLfg+Tl5Zlz64kocaWlpSgtLXWvl1DBq2pN/WUR\neRPAkljX8xaYEFHLyM/Pv2TBU7zNRBN6SS8i0cuY7gdgb4NJRGmhKcNy7wCYBKCniFQCeBHAZBEZ\ng7pP68sB/CapR0lELcIteFWdGePbbzXlzgcOHBg3u/baa83b7tq1y8ytMX4AuPrqq83c60nurbc/\neTLWwMVPcnJyzLyoqMjMz507Z+beGPWYMWPM3FsLXlFRYeYAUFJSYubemv2MjAwzf/bZZ83c+xuP\nGzfOzL3HgLde3eOtp/ceY975qa2tveJj4kw7ooCw4IkCwoInCggLniggLHiigLDgiQLCgicKSFL7\n0nfvHn9NzY4dO8zbeuPA3nruBQsWmLk1RwDw+8pv3LjRzL21zgUFBWa+cuVKM/fWkj/yyCNmPnfu\nXDP/4IMPzBzwz6HXl96bS7F7924zf/DBB83c+xt6PRe8uSDeXIkLFy6Y+T333GPmXk+CpsyVaIzP\n8EQBYcETBYQFTxQQFjxRQFjwRAFhwRMFhAVPFBBR1eTcsYhaY7neWmRvDNZbz/3111+buTcGa80h\nAPwxaG+cfsSIEWaemZlp5l5PdO/4y8rKzNxbyw74e8h752DixIlm7j02T5w4YeZeb35v//n+/fub\nuTfXYtCgQWZ+8OBBM/fW+1s9GWbNmgVVvWxBP5/hiQLCgicKCAueKCAseKKAsOCJAsKCJwoIC54o\nIEldD2/19W7Txv63xlvL7I3je+PcZ8+eNfN27dqZ+bFjx8x85sxY7fx/4q219nqaez3Tvb743lrq\nfv36mTngn4M5c+aYuTfO7J0jr/f+mjVrzNzbv90b5+/SpYuZb9682cw9hw8fNnPv/MTCZ3iigLDg\niQLCgicKCAueKCAseKKAsOCJAsKCJwqIuR5eRHIB/A1AbwAK4L9V9b9EJBPAPwH0B1ABYIaqnmp0\nW/3888/j3re3Xj0vL8/Mjx8/buabNm0y89tvv93M165da+aFhYVm/sUXX5i5NwbsjaOfOnXKzIcM\nGWLm3v72Z86cMXMAGDp0qJl768lXr15t5kVFRWbujVN7fd2XLFli5rm5uWY+fPhwM/f+hj/++KOZ\ne+dv+/btcbMZM2YktB6+FsAzqpoPYDyA34rIcADPA1ihqkMBrIp8TURpzix4VT2iqpsjl88AKAOQ\nA+BeAPMiV5sH4L5kHiQRtYwmv4cXkTwABQDWAchS1epIVA0gq8WPjIhaXJPm0otIBoAFAJ5S1W+j\n53GrqopIzA8CovcvKygowNixY5t3tEQUU2lpKUpLS93ruQUvIu1QV+x/V9VFkW9Xi0gfVT0iItkA\namLd9tFHH72CQyaiROXn51/SVLS4uDjm9cyX9FL3VD4XwHZVfS0qWgzg4cjlhwEsanxbIko/3jP8\nLQBmASgRkfpxrhcA/BHAv0TkUUSG5ZJ2hETUYsyCV9XPEP9VwBTvzq1xUm89e3V1tZl7fdO9tcrW\nGCYAPPDAA2burUX2eqJ78wi8vvQ9e/Y0c28M1+spv2HDBjMH/L/RsGHDzLygoMDMa2trzbxjx45m\nvmLFCjP3eHNFvL70N954o5l7cz26du1q5t75iYUz7YgCwoInCggLniggLHiigLDgiQLCgicKCAue\nKCBJ3R9+/vz5cfPly5ebtx83bpyZe3tvZ2RkmPmhQ4fM3Oubv23bNjOfPHmymT/11FNm7vVs93qy\ne/MQpk6dauZez3nAX09+/fXXm7nXk8DrabBlyxYz79Wrl5lPmDDBzA8cOGDmVVVVZu71HPD2TvD2\nRjh9+nTc7IknnuD+8EShY8ETBYQFTxQQFjxRQFjwRAFhwRMFhAVPFJCkjsO//PLLcXNvvbfXE9xb\nz37TTTeZed++fc3c6qkPAN98842Ze/vPe+P877//vpmPHj3azKdPn27m3nr4pqwlnzRpkpl7e9j/\n8MMPzcp3795t5l5feG89uTeOfvToUTP3ej54ewt4fe+tuSRz5szhODxR6FjwRAFhwRMFhAVPFBAW\nPFFAWPBEAWHBEwWkSXvLJer777+Pm3l7Y3v7q992221mXlFRYeZe3/js7Gwz98aIvbXW3t7lAwcO\nNHOv53u/fv3M3OsH4PVUB/y+9N4e817v/KVLl5r54MGDzdzriWCtJwf8cXDvMez1rff2VsjLyzNz\nb718LHyGJwoIC54oICx4ooCw4IkCwoInCggLniggZsGLSK6IrBaRUhHZJiJPRr7/BxE5KCKbIv9N\nS83hElFzeOPwtQCeUdXNIpIBYKOIrACgAF5V1VetG3fo0CFu5o0Te2O4q1evNnNvDNTjrbUeNWqU\nmXtrpXv06GHm3v7v3t7hJSUlZn7x4kUz99aSA37f9m7dupn5hQsXzNzr2+6Ns3s9Cbz17vv27TPz\n8ePHm/mHH35o5t5cDa/v/uHDh808FrPgVfUIgCORy2dEpAxA/SPB7m5ARGmnye/hRSQPQAGAtZFv\n/U5EtojIXBHpnoRjI6IW1qSCj7ycLwbwlKqeAfAXAAMAjAFwGMCfk3aERNRi3Ln0ItIOwAIA/6Oq\niwBAVWui8jcBxNxkbOXKlQ2XBw4c6M4PJ6LE7N271/3MAXAKXuq6EM4FsF1VX4v6fraq1n9icD+A\nrbFuP2XKlCYfMBElbtCgQZdssLpq1aqY1/Oe4W8BMAtAiYjUb+X5ewAzRWQM6j6tLwfwm+YeMBEl\nn/cp/WeI/T5/WXIOh4iSKanr4SdOnBg3O378uHnbwsJCM1+/fr2ZDx061My99ezeOHdNTY2Ze2O8\nXu715d+/f7+Znzhxwsy9vc179+5t5oA/lr9nzx4z9/ag93rrf/bZZ2buHZ93DgcMGGDmXl95qx8E\nAHh7Qnh/A+/8xcKptUQBYcETBYQFTxQQFjxRQFjwRAFhwRMFhAVPFJCkjsNba7K9vblHjhxp5l5f\nda9nurcef9kye25R9+72AsHm9jT3+gG0bdvWzNetW2fm3u/vzTMA/HPgrXf3/sbeenBvLoP3OxYV\nFZm511PAewx6Fi9ebOaTJk0y806dOl3xz+QzPFFAWPBEAWHBEwUkZQW/a9euVP2ohHz99detfQim\nzZs3t/YhmNL9/JWWlrb2IZi8vQ5bCgs+YtOmTf6VWpH3AVZrS/fzl+4F721+2lL4kp4oICx4ooCI\ntyY34TsWSc4dE1GTqOplreSTVvBElH74kp4oICx4ooCkpOBFZJqI7BCR3SLyXCp+5pUQkQoRKYls\njGk3y0vN8bwlItUisjXqe5kiskJEdonIh62520+c40uLDUaNDVDT4vy19gatSX8PLyJtAewEMAVA\nFYANAGaqallSf/AVEJFyAONU1e78mCIiMhHAGQB/U9VRke/9CcAxVf1T5B/Na1T1+TQ6vhcBfOtt\nMJqCY+sDoE/0BqgA7gPwa6TB+TOObwZScP5S8QxfCGCPqlaoai2A+QB+mYKfe6XSZnNMVV0DoPFS\nsHsBzItcnoe6B0mriHN8QBqcQ1U9oqqbI5fPAKjfADUtzp9xfEAKzl8qCj4HQGXU1wfx0y+YLhTA\nShH5SkQeb+2DiSNLVevX/FYDyGrNg4kjrTYYjdoAdR3S8Py1xgatqSj4n8O43y2qWgDgTgC/jbxk\nTVta9z4s3c5rWm0wGnm5vAB1G6B+G52lw/lrrQ1aU1HwVQCid1XIRd2zfNqo3ydPVY8CWIi6tyHp\npjry/g8ikg3A71CRQqpaoxEA3kQrnsOoDVD/Xr8BKtLo/MXboDUV5y8VBf8VgCEikici7QH8CoDd\n6iOFRKSziHSNXO4C4A7E2RyzlS0G8HDk8sMAFhnXTblIEdWLu8FoCo4j5gaoSJPzZ23QGnW1pJ2/\nlMy0E5E7AbwGoC2Auar6n0n/oU0kIgNQ96wO1LX8+kdrH5+IvANgEoCeqHu/+W8A/hfAvwD0A1AB\nYIaq2nsdpe74XgQwGXUvRxs2GI16z5zKY5sA4FMAJfjpZfsLANYjDc5fnOP7PYCZSMH549RaooBw\nph1RQFjwRAFhwRMFhAVPFBAWPFFAWPBEAWHBEwWEBU8UkP8H8cLeIV4V1NwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10cac0940>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPwAAAD8CAYAAABTq8lnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGJdJREFUeJzt3Xtw1OXVB/DvAcPVIPcEQjBEw1VujVAQBASsoIANnWLp\nOEOdvtjp9K2tnc708sfb9rVTOo69jDOWji10aF8qESmoUxUCiMBQJIGQcCcIUQgkSLmLF8Dz/pEl\nTcLuOXE3m6zzfD8zjrv57m5+2ezht5vnec4jqgoiCkOb1j4AImo5LHiigLDgiQLCgicKCAueKCAs\neKKAxF3wIjJDRA6KSIWI/Kg5D4qIkkPiGYcXkbYADgGYDqAKQDGA+ap6oN5tOMBP1IpUVRp/7ZY4\nH2ssgCOqWgkAIrICwMMADtS/0S9/+cu6yxs2bMC0adPqrl+8eNH8Bg8//LCZFxUVmXlFRYWZz549\nu8H1lStX4qtf/Wrd9fbt25v337dvn5mnpaWZ+SeffGLmgwcPbnD9xRdfxLx58+quV1dXm/e/cOGC\nmY8fP97M33nnHTMHgKqqqrrLmzZtwpQpUxrkd911l3n/QYMGmXlhYaGZT5w40cwPHz5cd/n111/H\nzJkzG+T5+fnm/c+cOWPm/fv3N/PnnnvOzIcMGVJ3ee3atXjggQca5B999JF5/y5dusTMnnjiiahf\nj/ctfRaA4/Wun4h8jYhSWLwFz7frRJ9D8b6lrwKQXe96NmrP8g1s2LCh7nKHDh3i/FYtY+jQoa19\nCKZhw4a19iGYcnJyWvsQTHfeeWdrH4LpjjvuSOj+FRUV7sdYIP6CLwGQJyI5AE4CeATA/MY3qv+Z\nPdWlekGl+vGlesHn5eW19iGYEv0HKS8vr8HP+MYbb0S9XVwFr6rXROS/AawF0BbAkvp/oSei1BTv\nGR6q+jqA163bdO/ePWZWUFBgPv6rr75q5keOHDHzdu3amfnGjRvNfNKkSWZ+4sRNn2Aa8P6Caz03\nAPDhhx+a+dGjR828Y8eOZt61a1czP3nypJkDQN++fc28d+/eZl5TU2Pm3rua/fv3m3n9UZdo3n33\nXTNfvXq1mWdl2X+n9l4D3kiQ9zvwXiPRcKYdUUBY8EQBYcETBYQFTxQQFjxRQFjwRAFhwRMFJO5x\n+KY4depUzGzTpk3mfb/4xS+aeXp6upl7Y8ClpaVmXllZaebezLLz58+bea9evRJ6/H79+pm5N0bt\njfF6Y9QAMGvWLDNv27atma9atcrMMzMzzVzkptWfDWzdutXMvRWB3s/31ltvmXmnTp3M3FtN+MEH\nH5i5t+I0Gp7hiQLCgicKCAueKCAseKKAsOCJAsKCJwoIC54oIEkdh7e6jHjj1N44sjcGe+XKFTP3\nura+9NJLZv7QQw+ZeUZGhpk//fTTZj537lwz37Vrl5kvXLjQzHfs2GHmXkdYwO9J4H2Pzp07m7nX\nQt3r/OvNJfCO/+rVq2Y+duxYM/c6G7dpY59vx40bZ+YlJSVmHvV7fuZ7ENHnFgueKCAseKKAsOCJ\nAsKCJwoIC54oICx4ooAkdRze2sG0/jZU0Xg9yUeMGGHm3s6f3rY899xzj5l7a6m9nuTezqm33Xab\nmU+ePNnMvTHk4uJiM7///vvNHAAuXbpk5t4Ot972Xp9++qmZe3MtvN+h1xPA29tg5MiRZt6zZ08z\n9/YmWLlypZl7u8tGwzM8UUBY8EQBYcETBYQFTxQQFjxRQFjwRAFhwRMFRLw1x+adRSoBXARwHcBV\nVR1bL9Nly5bFvO97771nPra3VrpDhw7esZl5ly5dzNwb4/TGeD/++GMzLysrM/MhQ4aYeVVVlZmf\nPXvWzBPdux3we+efO3fOzJ955hkzz87ONnPvZ/D2T/fWw3vr0b/+9a+bubUvAwAcOnTIzL2+89be\nDTNmzICq3lQEiU68UQBTVNV+dRFRSmiOt/T2qZSIUkaiBa8A1otIiYjYPZWIqNUl+pZ+gqqeEpFe\nAIpE5KCqbrkRrl69uu6GgwcPdj+XElF8ysrKUF5e7t4uoYJX1VOR/78vIqsBjAVQV/AFBQWJPDwR\nNdHIkSMbLOZZvnx51NvF/ZZeRDqJSHrkcmcAXwKwJ97HI6LkS+QMnwFgdWT46xYAy1V1XbMcFREl\nRdwFr6rHAIyybmP1lvfWMnft2tXMrZ73gD+O7I3ze3ubFxYWmvmAAQPM3JsncPnyZTNPT083c29v\ndW8c31vLDQDDhw83c29/88WLF5v5li1bzHz79u1m7v2OvXH+a9eumbnX996bB+DN1bh+/bqZsy89\nEZlY8EQBYcETBYQFTxQQFjxRQFjwRAFhwRMFJKl96a3e6NOmTTPv6+393bFjRzP3eo57vHFsb4z5\n+PHjZv7ggw+audez3Ftr3qlTJzP3xpC9teCA/xx7vfW3bt1q5mlpaWbu9f735hrs2rXLzJ977jkz\nP3DggJmfPn3azL2fz3v+vH0BouEZniggLHiigLDgiQLCgicKCAueKCAseKKAsOCJApLUcfg2bWL/\ne+Ltz3706FEzHzXKXIrvjhF7a4nvvvtuM58xY4aZ796928y9/mM9evQwc28tt7fe/v333zfzX/3q\nV2YO+HsLTJ061czvu+8+Mz9z5oyZ33rrrWburdcfM2aMmX//+983c68vvdfzweorD/jr6b25FtHw\nDE8UEBY8UUBY8EQBYcETBYQFTxQQFjxRQFjwRAFJ6jj8oEGDYmbe/uUjRowwc6vnPeCvN/fWEns9\nw73v742DHzt2zMwnTJhg5t4YtTfO7/Xt99bLA/56ba9ngDdO7fXG99abP/vss2b+u9/9zsy99fQr\nVqww88cff9zMKysrzdzrSeD9/NHwDE8UEBY8UUBY8EQBYcETBYQFTxQQFjxRQFjwRAFxx+FFZCmA\nhwCcVtXhka91B1AI4HYAlQDmqer5xvfdvHlzzMf11hJ769lHjx5t5t7+8mVlZWbeu3dvM8/IyDBz\nT79+/czc61m+ZMkSM+/bt6+Zjx8/3sxLS0vNHABOnTpl5t/4xjfM/Pz5m14yDdxzzz1m7vW1/+EP\nf2jmf/rTn8w8PT3dzEeOHGnm3jyD6dOnm7nXs8B7jUTTlDP8XwA07vbwYwBFqjoQwIbIdSJKcW7B\nq+oWAI23OZkDYFnk8jIAX27m4yKiJIj3M3yGqt6Ym1kDILH3t0TUIhKeS6+qKiIaLav/OTkjI8Pd\nr42I4rNnzx7s2bPHvV28BV8jIpmqWi0ifQBEncXv/VGDiJrH8OHDGzTtjLWwJ9639K8AWBC5vADA\nmjgfh4hakFvwIvICgG0ABonIcRF5DMCvAdwvIocBTI1cJ6IU576lV9X5MSJ7EBFAXl5ezMxbT+6t\ntfZ6km/bts3Mb7nF/tG9tdzeevIhQ4aYeVFRkZl7+8tbvQYA//k5cuSImX/hC18wcwD49NNPzfza\ntWtmvm7dOjPPzc01c+v1BfjPwdChQ83cOz7vb1L//Oc/zTwrK8vMDx8+bOadO3c282g4044oICx4\nooCw4IkCwoInCggLniggLHiigLDgiQKS1L70ly9fjpldv37dvK83Dl9cXGzm3hixtzd4YWGhmc+Z\nM8fMvXH69u3bm/mAAQPM3Ju23K5dOzN/6qmnzPzo0aNmDgAzZjReNd2Qt7/5/PmxpnjUWrlypZl7\n+6P36dPHzF977TUzHzt2rJn/8Y9/NPMLFy6YeZcuXcx81KhRZu71E4iGZ3iigLDgiQLCgicKCAue\nKCAseKKAsOCJAsKCJwpIUsfhrTXV3hhlfn6+mXv7y3vr3b390++8804z9/rSe+PsGzZsMPMOHTqY\n+cCBA828e/fuZj5x4sSEvj/gj4O//PLLZt6rVy8zf/TRR83c66ngvUa8vveHDh0y84ULF5q59zv2\n5lKcOXPGzCdMmGDm0fAMTxQQFjxRQFjwRAFhwRMFhAVPFBAWPFFAWPBEAUnqOPzFixdjZt56bW89\n9t69e8188uTJZu6NUx84cMDMvb73I0aMMPPBgwebedu2bc3cW+/vzTOorKw086b0pe/du7eZT5ky\nxcy936HX8+CTTz4x8/fee8/MS0tLzXzmzJlm7s1VGD16tJl7e8HNnTvXzL2fLxqe4YkCwoInCggL\nniggLHiigLDgiQLCgicKCAueKCDuOLyILAXwEIDTqjo88rWfA/gvAO9HbvYTVX2j8X1PnjwZ83G7\ndetmft9z586Z+WOPPWbmO3fuNPN3333XzHNycszc27vc2zt8+vTpZv7CCy+Y+b59+8zc+/lnzZpl\n5mlpaWYO+D/jxo0bzdwb5/bmEljzPAB/nkBBQYGZq6qZf/DBB2buzfXwei54fedPnz5t5tE05Qz/\nFwCNdxxQAL9V1dGR/24qdiJKPW7Bq+oWANFOt9L8h0NEyZTIZ/jvikiZiCwRka7NdkRElDTxzqVf\nDOB/I5efAvAbAN9sfKM333yz7nJOTo67XxoRxef48eM4ceKEe7u4Cl5V6/5aICJ/BvBqtNvdd999\n8Tw8EX1G2dnZyM7Orru+ffv2qLeL6y29iNTflrMAgL3sh4hSQlOG5V4AMBlATxE5DuBnAKaIyCjU\n/rX+GIBvJfUoiahZuAWvqtE28V7alAfPy8uLmXnj4P369TPz5cuXm/nUqVPN3OtpfurUKTP39h5/\n5513zPz69etmftddd5n5M888Y+be3uXeevcrV66YOQCsW7fOzLOyssy8/t944jmG4cOHm7nXk8Bb\nT75mzRoznzRpkpl7feW9vRl2795t5uPGjTPzaDjTjiggLHiigLDgiQLCgicKCAueKCAseKKAsOCJ\nApLUvvTp6ekxs44dO5r3jTU18Aav57mnb9++Zu71bffGcBcvXmzmXl96b+/0ZcuWJXT/jz/+2Mx3\n7Nhh5gBw8OBBM2/fvr2ZV1VVmfns2bPN3Ovd7/Wt9/a3HzVqlJmvXbvWzL15At7+8KtWrTLznj17\nmnk0PMMTBYQFTxQQFjxRQFjwRAFhwRMFhAVPFBAWPFFAkjoOb/VOnzGjcefrhqwxfMDfP94bpx80\naJCZez3Pvf3jH3nkETP3xrC9vcU93hjvv//9bzP3+gUAcHuoTZs2zcwzMjLM/Nq1a2beuXNnM/d6\nGnh93cvKyszcew21aWOfT3ft2mXm8+bNM/Om/I5uOqbPfA8i+txiwRMFhAVPFBAWPFFAWPBEAWHB\nEwWEBU8UkKSOw1v7c//iF78w7+vt//7hhx+audcT3Vvv7a3Hv/fee828urrazL1x9pKSEjMfM2aM\nmVdUVJi5d3zePAcAWLhwoZl7PQUmT55s5t7+6W+8Ye9SnpOTY+be/vFnz541c28egfca7dGjh5mX\nlpaa+bFjx8w8Gp7hiQLCgicKCAueKCAseKKAsOCJAsKCJwoIC54oIOY4vIhkA/grgN4AFMDzqvqs\niHQHUAjgdgCVAOap6vnG97f2gH/yySfNA6upqTFzry/8okWLzDwzM9PM77jjDjMvLy83c2+MuF+/\nfmYuImauqmaeaN//pqzH99Zjf/vb3zZzr6+89foBEu+97/UEGD9+vJkXFxebeX5+vpl7r7H169eb\nebdu3cw8Gu8MfxXAk6o6DMA4AN8RkSEAfgygSFUHAtgQuU5EKc4seFWtVtXdkcuXARwAkAVgDoAb\nW58sA/DlZB4kETWPJn+GF5EcAKMBvA0gQ1VvvOeuAWDPMSSilNCkufQiciuAVQC+p6qX6n++VFUV\nkagfKDdt2lR3OScnx53bTETxOXr0aJPWP7gFLyJpqC32v6nqmsiXa0QkU1WrRaQPgKjdABPd8JGI\nmiY3Nxe5ubl11zdu3Bj1duZbeqk9lS8BsF9Vf18vegXAgsjlBQDWNL4vEaUe7ww/AcCjAMpF5MZa\nvZ8A+DWAF0Xkm4gMyyXtCImo2ZgFr6pbEftdwHTvwQcOHBgz83p+e+vVv/KVr5j5pUuXzNxba+zt\nXe71RPfWcs+dO9fMX3vtNTP31mKfPHnSzOfMmWPm3t7qgD8XwttbwJsLcP36dTP31oO3a9fOzC9c\nuJDQ/b1+AN48AG8eQffu3RN6/Gg4044oICx4ooCw4IkCwoInCggLniggLHiigLDgiQKS1L70a9eu\njZnNnj3bvK83Bnv48GEz98YwvZ7ht9xiPzXeOLe31vnvf/+7md9+++1m/vzzz5t5z549zdzb29zr\nWw8ADz74YEKP4e3f3qFDBzP3fkezZs0y81jTT2/wfsfeOPjSpUvN3PsdeHMhvNdYNDzDEwWEBU8U\nEBY8UUBY8EQBYcETBYQFTxQQFjxRQJI6Dm+NpXvj5AsWLDBzbxze69vujWF6j+/tD+/1F/N+vp07\nd5r5bbfdZuYnTpwwc29v9P79+5s54K+Hb9PGPp94rwGvb70316CwsNDMx4wZY+bea6CoqMjMvZ4H\nXs8Fbx6A128gGp7hiQLCgicKCAueKCAseKKAsOCJAsKCJwoIC54oIEkdh+/SpUvMbNu2beZ9vZ7j\neXl5Zt6+fXszP3066u5YdQoKCsy8oqLCzL29u73948+dO2fm2dnZZt6jRw8z9/ZenzRpkpkDwMWL\nF83cGwf3+r57vf29/d2HDRtm5v/617/M3JtH4PVsWLFihZl7W7F54/BXrlwx82h4hicKCAueKCAs\neKKAsOCJAsKCJwoIC54oIGbBi0i2iLwpIvtEZK+IPBH5+s9F5ISIlEb+m9Eyh0tEiRBr3biIZALI\nVNXdInIrgJ0AvgxgHoBLqvpb4766aNGimI+9f/9+88CsveUBfy21tz+81/fdW+t99913m7k1BwEA\nysrKzNx7frz17GlpaWbuzVPw5gEAwMiRI83c+x15cxG8uQRXr141c6+v/dChQ838H//4h5l7r6EB\nAwaYeWZmppnv2bPHzPft2xcz+8Mf/gBVlcZfNyfeqGo1gOrI5csicgBAViS+6cGIKLU1+TO8iOQA\nGA1ge+RL3xWRMhFZIiJdk3BsRNTMmlTwkbfzLwH4nqpeBrAYwAAAowCcAvCbpB0hETUbdy69iKQB\nWAXg/1R1DQCo6ul6+Z8BvBrtvuvXr6+7nJubi9zc3ESPl4iiqKqqcnvkAU7Bi4gAWAJgv6r+vt7X\n+6jqjZ0ACwBE/evC9OnTm3zARBS/rKwsZGVl1V0vKSmJejvvDD8BwKMAykWkNPK1nwKYLyKjACiA\nYwC+legBE1HyeX+l34ron/NfT87hEFEyJXU9vDUW/oMf/MC874EDB8z87bffNnOvZ/dbb71l5t4Y\nrtf33lsrvXr1ajN/4IEHzNwb5z9+/LiZJ7rWGwAOHjxo5r169TLz/Px8M/d6Cnh9472eCdY4NgB8\n7WtfM3NvLkXHjh3NfPPmzWaekZFh5n369DHzaDi1liggLHiigLDgiQLCgicKCAueKCAseKKAsOCJ\nApLUcXhr/+368+yj8dZje2OQ5eXlZl5/GmI0Xt96r2++t9788ccfN/NOnTqZeXFxsZmPGzfOzD/6\n6CMzX7NmjZkD/v7nXt/0Xbt2mbnX+3/q1KlmXl1dbeZnzpwxc2+c3Zsn4L3Gvb77Xbvai1C932E0\nPMMTBYQFTxQQFjxRQFqs4Pfu3dtS3youXg+71uZ9nmxt3t9MWpvXI7C1eX+vaC4s+AhvsUlrS/WC\n9xoutjYWfC2+pScKCAueKCBmX/qEHlgkOQ9MRE0SrS990gqeiFIP39ITBYQFTxSQFil4EZkhIgdF\npEJEftQS3/OzEJFKESmPbIy5IwWOZ6mI1IjInnpf6y4iRSJyWETWteZuPzGOLyU2GDU2QE2J56+1\nN2hN+md4EWkL4BCA6QCqABQDmK+qdpfKFiQixwDkq+rZ1j4WABCRewFcBvBXVR0e+drTAM6o6tOR\nfzS7qeqPU+j4fgZng9EWOrZYG6A+hhR4/hLZoLU5tMQZfiyAI6paqapXAawA8HALfN/PKmU2x1TV\nLQAaLxecA2BZ5PIy1L5IWkWM4wNS4DlU1WpV3R25fBnAjQ1QU+L5M44PaIHnryUKPgtA/WlsJ/Cf\nHzBVKID1IlIiIgtb+2BiyFDVmsjlGgB2D+PWkVIbjNbbAPVtpODz1xobtLZEwX8exv0mqOpoADMB\nfCfyljVlae3nsFR7XlNqg9HI2+VVqN0AtcEGCanw/LXWBq0tUfBVALLrXc9G7Vk+ZdzYJ09V3wew\nGrUfQ1JNTeTzH0SkDwC7Q0cLU9XTGgHgz2jF57DeBqh/u7EBKlLo+Yu1QWtLPH8tUfAlAPJEJEdE\n2gF4BMArLfB9m0REOolIeuRyZwBfQozNMVvZKwAWRC4vAOC3pGlBkSK6IeYGoy1wHFE3QEWKPH/W\nBq31bpa0569FZtqJyEwAvwfQFsASVV2U9G/aRCIyALVndaC25dfy1j4+EXkBwGQAPVH7efN/ALwM\n4EUA/QFUApinqudT5Ph+BmAKat+O1m0wWu8zc0se20QAmwGU4z9v238CYAdS4PmLcXw/BTAfLfD8\ncWotUUA4044oICx4ooCw4IkCwoInCggLniggLHiigLDgiQLCgicKyP8DqxLGECbnNHcAAAAASUVO\nRK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10b5005c0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPwAAAD8CAYAAABTq8lnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGVNJREFUeJztnXmQVNXZxp/XAUVQQQRh2CWCGAiLe6kERFwIoKAVFGUJ\nQTTqx0cwoWJC5YullSjBWGgiLggEdYK4AILiwi6Iihh2ZA0DyDIgIMoii57vD5rJMHY/7zhDz7R1\nnl+Vle7+3Xv7cKef3Nt9znmPhRAghIiDk8q6AUKI0kOBFyIiFHghIkKBFyIiFHghIkKBFyIiih14\nM7vezFaa2Roz+92JbJQQIj1YcfrhzSwLwCoA7QFsBvAxgO4hhE8LbKMOfiHKkBCCFX6tXDGPdQmA\ntSGEXAAws5cA3Ajg04Ib9enTJ//xwoUL0apVq/znV111FX2DvLw86itUqED9rl27qL/pppuOez58\n+HDcc889+c/ffPNNuv/tt99O/QMPPEB98+bNqX/55ZePe75x40bUq1cv//mll15K97/55pupHzp0\nKPWHDx+mHgC6deuW/3jChAno2rXr93oPr41ZWVnUt23blvqxY8fmP/74449x8cUXH+e/+OILun/T\npk2pv+WWW6hfsWIF9bNnz85/PG/ePFx++eXH+ezsbLp/v379UroqVaokfb24t/S1AWwq8PyzxGtC\niAymuIHX7boQP0CKe0u/GUDdAs/r4uhV/jgWLlyY/7h8+fLFfKvSofDtXqZRuXLlsm4CpUmTJmXd\nBEqtWrXKugmUunXr+hsR5syZg7lz57rbFTfwCwA0MrMGALYAuAVA98IbFfzOnuko8CXj/PPPL+sm\nUGrXzuxvnCUNfOvWrdG6dev850OGDEm6XbECH0I4Ymb/A+AdAFkARhb8hV4IkZkUq1uuSAc2C5s3\nb07pp02bRvdfvnw59evXr6f+vvvuo37JkiXU16lTh3rvF2SvfStXrqT+zDPPpH779u3Us3MPAL16\n9aL+73//O/UA0K5dO+rr169PvfdvSPVL8zFyc3Opb9iwIfVHjhyhvuAVMxmPPvoo9TNnzqTe+wwO\nGjSIenZX2rdv36TdchppJ0REKPBCRIQCL0REKPBCRIQCL0REKPBCRIQCL0REpLUffurUqSn9qlWr\n6P6fffadkbrH4fWxerz++uvUX3DBBdT/6Ec/ov7cc8+lfvLkydR7fdiTJk2i3usjX7t2LfWdO3em\nHvDHOhScfZiMnJwc6gcOHEi99zcYMGBAiY7vZeO0006jfufOndR7I1HPPvts6gvPqCzIgAED1A8v\nROwo8EJEhAIvREQo8EJEhAIvREQo8EJEhAIvREQUt+JNkTh06FBK9+WXX9J9vYqkVatWpd4r9+PN\nZWZjCACgQYMG1HtVX3/84x9TP2zYMOpHjx5N/Z49e6j3+oifeeYZ6gF/Tn25cvzj5c039/q5582b\nR32NGjWo96oIzZo1i/pt27ZRX7FiReoLV6ktjFf1tjhVmnSFFyIiFHghIkKBFyIiFHghIkKBFyIi\nFHghIkKBFyIi0jof/vHHH0/p58+fT/f36rJ7/cgdOnSg3quJ/tvf/pZ6r596wYIF1Ddu3Jh6r678\nuHHjqO/Zsyf1n3/+OfXeXGwA2L9/P/VeP3Tv3r2pv+uuu6j3xgFs2LCBeq/2/vDhw6n3zpG3vJr3\nGfT+RmwsR5s2bTQfXojYUeCFiAgFXoiIUOCFiAgFXoiIUOCFiAgFXoiIKNF8eDPLBfAlgG8AHA4h\nXFLQV6hQIeW+P/vZz+ixvbXBvZrkH374IfXefHxvHMDWrVupb9asGfVeH6v373vxxRep9+aKe+ME\natasST0AvPfee9R7YxX27dtH/e7du6nPzs6mfsaMGdRfdtll1J966qnUf/LJJ9R7axt4NRO8mhAr\nV66kPhklLYARALQNIewq4XGEEKXAibil/85oHiFEZlLSwAcA08xsgZn1OxENEkKkj5Le0l8RQthq\nZtUBTDWzlSGEOcdkwfXTGjdujPPOO6+EbyeESMayZcuwbNkyd7sSBT6EsDXxvzvMbAKASwDkB74o\nCxIKIUpOs2bNjvuhONXkqmLf0ptZRTM7PfG4EoBrASwt7vGEEOmnJFf4GgAmmNmx4+SEEN49Ia0S\nQqSFYgc+hLAeQEu2zZYtW1K6/v370+MvX76c+kWLFlF/4MAB6g8ePEh9jx49qPf6UCtVqkR9+/bt\nS7S/V5e+WrVq1D/22GPUF6WPt3nz5tQPGTKE+pNO4jeYbdq0oT4rK4v67t27U++NxWjZkn683br2\nXj/+v/71L+q9z6h3/pLu8733EEL8YFHghYgIBV6IiFDghYgIBV6IiFDghYgIBV6IiEjr+vCdOnVK\n6RYvXkz3Pfnkk6l/6623qPf6YMePH089azvg95N7/dje+vJ79+6l3puXsGnTJuq9+e5eTXfArxnA\n6iEAQMOGDam/7bbbqJ8yZQr1V1xxBfVLl/KBoW+88Qb13vrtXt39r7/+mnqvHoBXUyEZusILEREK\nvBARocALEREKvBARocALEREKvBARocALERFpXR+e1Ybv06cP3f+RRx6hft26ddR/8MEH1Hv92GvW\nrKF+8ODB1Hvz5b36Y17Nda8uvtfH7NVM98YBAMCkSZOov/fee6n/7LPPqF+9ejX1F154IfWzZ8+m\n3ptvfuedd1Lv1fb3PkO1atWi/qKLLqJ+1KhRKV1OTo7WhxcidhR4ISJCgRciIhR4ISJCgRciIhR4\nISJCgRciItLaD3/33Xen9LVr16b7e32QM2fOpL5cOT7Vf8eOHdS3a9eOem9t8DvuuIP6d9/la3Z4\n/ezXXXcd9V5d+pycHOq99e0Bv6/eq1s/cuRI6uvWrUv9+vXrqfdqDpx22mnUr127lvrWrVtT760/\nP3DgQOoHDRpEPcvX0qVL1Q8vROwo8EJEhAIvREQo8EJEhAIvREQo8EJEhAIvRES4denNbBSAjgC2\nhxB+knitKoBxAOoDyAXQLYTwReF9WV+3V1Pc6+P0+tEHDBhAvTdf/YknnqD+0KFD1Hvt69mzJ/XT\np0+n3quLP3nyZOq9fvqi9MNPnDiR+ry8POrNvtNNfBxebf8WLVpQ782nX758OfXeZ+iLL77zkT8O\nb20A72/wzTffUM/GeqRqe1Gu8KMBXF/otfsBTA0hNAYwPfFcCJHhuIEPIcwBUHgJjBsAjEk8HgOg\nywlulxAiDRT3O3yNEMKx+7U8ADVOUHuEEGmkxGvLhRCCmSUdkF/wO1L16tVx9tlnl/TthBBJWLNm\njTv2Hyh+4PPMrGYIYZuZZQPYnmyjpk2bFvPwQojvQ6NGjdCoUaP852+//XbS7Yp7Sz8JwLGlMXsD\n4D/XCiEyAjfwZjYWwDwA55nZJjPrA+ARANeY2WoA7RLPhRAZjntLH0JItdB6e29fNqfcu93/9ttv\nqffmi0+YMIF6r5/8qquuot77PcKr2e7N5/f6YMuXL0/9ueeeS32TJk2o9+raA0D9+vWp37496Te9\nfLp04Z07Xu3+xYsXU+/VenjooYeof+WVV6g/88wzqT/rrLOor1KlCvXeOAK27kMqNNJOiIhQ4IWI\nCAVeiIhQ4IWICAVeiIhQ4IWICAVeiIgo8Vh6Bqsb/vnnn9N9vfnynTp1on7RokXU9+rVi3pvbfHr\nry88Y/h4vLr6o0ePpt6bj75r1y7qvT7ar776ivrf/OY31APAuHHjqJ82bRr1P//5z6n3xlp4YzVu\nvfVW6lMNPz3GySefTP3VV19NvUdJ6+p78+mToSu8EBGhwAsREQq8EBGhwAsREQq8EBGhwAsREQq8\nEBGR1n54Vvv99NNPp/t687W9+ext27al3qv/demll1L/0UcfUf/II7wmyOOPP079kSNHqL/kkkuo\nr1mzJvWnnnoq9V5dfsBfv92bkz927FjqvbEWWVlZ1HtjKWrVqkW9N9Zi1KhR1J9//vnU/+pXv6K+\ncuXK1O/Zs4f6ZOgKL0REKPBCRIQCL0REKPBCRIQCL0REKPBCRIQCL0REpLUfvl69eimdV3N8xowZ\n1H/99dfUe3XvDxw4QH3Dhg2p/+c//0n9P/7xD+q9cQTe+vUPPvgg9d58dq+uvXd+AH++e+/evan/\n9a9/Tf1JJ/HrkTcOoE2bNtSPGDGCem8shDeOYPDgwdS/88471Ht1/YszH19XeCEiQoEXIiIUeCEi\nQoEXIiIUeCEiQoEXIiIUeCEiwu2HN7NRADoC2B5C+EnitQcA3AHgWGfy70MI3ynynZeXl/K43tre\nzZs3p97rp161ahX13trfd955J/VVq1al3lv/vVw5fuq9Pujs7Gzqq1evTv2+ffuo99ZWB/x+aK+m\ngVe731tbwKvd761f79VM6N69O/Ve3XhvrIm3vvx9991H/bBhw6hPRlGu8KMBFK4EEAA8FkJolfiP\nV/QXQmQEbuBDCHMA7E6i7MQ3RwiRTkryHb6/mS02s5FmVuWEtUgIkTaKO5b+KQDHBnM/BOBvAPoW\n3mju3Ln5j+vVq0fH1gshis/GjRuxadMmd7tiBT6EkD+q38yeAzA52XZXXnllcQ4vhPieFL6gzps3\nL+l2xbqlN7OCPxF3BbC0OMcRQpQuRemWGwugDYBqZrYJwJ8AtDWzljj6a/16AHeltZVCiBOCFaW/\ntVgHNgt9+vRJ6b355hUqVKB+//791B88eJD6s846i3pv/fQ6depQ79Wt79KlC/Xe2udr1qyh3qNV\nq1bUe3X7AaBixYrUe/PVvZoG3t/41Vdfpb5v3+/8rHQc69ato96r++7Nl/fGcjz99NPU79y5k/pU\nt+3HXAjhOz1pGmknREQo8EJEhAIvREQo8EJEhAIvREQo8EJEhAIvRESktS49mw/tzZXeu3cv9VOm\nTKH+jDPOoN6b6+zNtfbm2x86dIh6bz66Vy/gxhtvpN5bn/7999+n3lvbHQA6duxI/QcffEB9u3bt\nqJ8zZw71/fv3p/7555+n/j//+Q/1Xt33Fi1aUP+LX/yCem++u/cZYDUTTujQWiHEDxMFXoiIUOCF\niAgFXoiIUOCFiAgFXoiIUOCFiIi0zoffunVrSv/oo4/S/StXrkz9xo0bqfdqhnt1371+9EaNGlHv\nzef36up77c/NzaXeq2/mzVV/8sknqQeAoUOHUu/VJDjllFOoN+OFkSdNmkR9lSq8tmq1atWo92om\nrF+/nvqmTZtS/8ILL1Dv1dVn8+WffPJJzYcXInYUeCEiQoEXIiIUeCEiQoEXIiIUeCEiQoEXIiLS\nOh++Z8+eKZ1XF/3LL7+kfuDAgdS/9NJL1Hs10W+++Wbqn3vuOeqvvfZa6r1+dm8+f4cOHaj31j73\nxkH89Kc/pR7w66a//TZfRfzWW2+l3hvL4M3Z9+are+MEXnvtNeovu+wy6seNG0f9HXfcQf3UqVOp\n99aXT4au8EJEhAIvREQo8EJEhAIvREQo8EJEhAIvREQo8EJEBO2HN7O6AJ4HcDaAAODZEMITZlYV\nwDgA9QHkAugWQvii8P5//vOfUx7bm4uck5ND/axZs6hfsWIF9Z07d6beq1nurf3trS3u1d335svf\nf//91Hs12QcNGkR9p06dqAeAmjVrUl+rVi3qvX58rx/eO4cPPvgg9b1796beY9u2bdR74wBYvQgA\n2LFjB/XefP1keFf4wwAGhhCaArgMwL1mdj6A+wFMDSE0BjA98VwIkeHQwIcQtoUQFiUe7wXwKYDa\nAG4AMCax2RgAXdLZSCHEiaHI3+HNrAGAVgA+AlAjhJCXUHkAapzwlgkhTjhFGktvZqcBeA3AgBDC\nVwVrjYUQgpklLYw3YsSI/McXXHABLrzwwpK1VgiRlNzcXGzYsMHdzg28mZXH0bC/EEKYmHg5z8xq\nhhC2mVk2gO3J9u3Xr9/3aLIQorg0aNDguAlZqRbipLf0dvRSPhLAihDCsAJqEoBjP3H2BjCx8L5C\niMzDu8JfAaAHgCVmtjDx2u8BPALgZTPri0S3XNpaKIQ4YdDAhxDmIvVdQHvv4Fu2bEnpdu/eTffd\nv3+/d3iKt3b45MmTqb/hhhuo9/pIvbnSXj/6L3/5S+qrV69OvddH/PDDD1O/du1a6gHg6aefpr5X\nr17U79q1i3pvPvibb75J/fjx46n35qt7Yy0qVqxI/erVq6k/cuQI9d58+TfeeIP6ZGiknRARocAL\nEREKvBARocALEREKvBARocALEREKvBARkda69Gx9bm/9cq/uvNeH6vXhdunCJ/ht3550tHA+H374\nIfVVq1alvk6dOtTv27ePeq8P1lv/vUYNPt/Jq9sPAJdffjn1EyfyAZjenHtvfXVvvv0DDzxA/d13\n3029N07Aq+1fvnx56ocPH079mjVrqPfqESRDV3ghIkKBFyIiFHghIkKBFyIiFHghIkKBFyIiFHgh\nIiKt/fCbN29O6bx+3nLleNPy8vKov/7666n/9ttvqZ8/fz71WVlZ1HtzoRs1akS9N47AW7v80KFD\n1Ht9uE899RT1ANC8eXPqvfniF198MfVezYFKlSpRf+DAAepnzpxJ/Y033kh93759qZ89ezb1U6ZM\nof7ZZ5+lfsaMGdQnQ1d4ISJCgRciIhR4ISJCgRciIhR4ISJCgRciIhR4ISIirf3w9erVS+muu+46\nuu+SJUuo79GjB/XeXOa5c+dS79UkHz16NPXefPi33nqLem++/tChQ6lv2bIl9W3btqX+008/pR7w\n12f3xgJ4/dTvvPMO9eeccw713vrp3mfMOwdeXf5WrVpR36xZM+q9cQYsQwsWLEj6uq7wQkSEAi9E\nRCjwQkSEAi9ERCjwQkSEAi9ERNDAm1ldM5tpZsvNbJmZ/W/i9QfM7DMzW5j4j89FFUJkBF4//GEA\nA0MIi8zsNACfmNlUAAHAYyGEx9jOrB/zoosuom989dVXU//vf/+7RP62226j3quLf8opp1Dvzedn\ntQIA4KGHHqL+j3/8I/V79uyhftasWdR7c9kB4Nprr6XeW2P+/fffp759+/bUe/PF//KXv1A/adIk\n6itXrkz9PffcQ/28efOo9zh8+DD1O3fu/N7HpJ/KEMI2ANsSj/ea2acAaie0fe93E0KUKUX+Dm9m\nDQC0AnBsyZX+ZrbYzEaaWeolZoQQGUORAp+4nX8VwIAQwl4ATwE4B0BLAFsB/C1tLRRCnDDcsfRm\nVh7AawBeDCFMBIAQwvYC/jkAk5PtO3nyf19u3LgxzjvvvJK2VwiRhM2bN2PLli3udjTwZmYARgJY\nEUIYVuD17BDC1sTTrgCWJtu/c+fORW6wEKL41K5dG7Vr185//sknnyTdzrvCXwGgB4AlZrYw8dof\nAHQ3s5Y4+mv9egB3lbTBQoj04/1KPxfJv+fzuZ1CiIwkrfPhWV+6VxO8QYMG1C9cuJD6+vXrU79u\n3Trqu3fvTv348eOp99aX79ixI/VLlyb9lpTPsmXLqPf6cL2a6w0bNqQeAI5+40uNN1Zh+vTp1Hft\n2pX6m266iXpvPn2NGjWo9z5D3t/ojDPOoN6ru++N5WjRogX1ydDQWiEiQoEXIiIUeCEiQoEXIiIU\neCEiQoEXIiIUeCEiwkII6TmwWWDzlb1+Ym+ucvXq1akfPHgw9UOGDKH+9ttvp75Xr17U9+vXj/oj\nR45Q37hxY+qbNGlCvdcHPmLECOrvussfPPnKK69Qn5OTQ/0zzzxD/ZgxY6i/5pprqD948CD1WVlZ\n1Hs1C/bt20d9+fLlqW/Xrh312dnZ1LOxGN26dUMI4TsDJXSFFyIiFHghIkKBFyIiSi3wq1atKq23\nKhZbt271NypDirLWW1mSm5tb1k2gZPrnb9GiRaXyPgp8gm3btpV1EygrV64s6yZQNmzYUNZNoKxe\nvbqsm0BZvHhxqbyPbumFiAgFXoiISGs/fFoOLIQoEsn64dMWeCFE5qFbeiEiQoEXIiJKJfBmdr2Z\nrTSzNWb2u9J4z++DmeWa2ZLEwpjzM6A9o8wsz8yWFnitqplNNbPVZvZuWa72k6J9GbHAKFkANSPO\nX1kv0Jr27/BmlgVgFYD2ADYD+BhA9xBCxowkMbP1AC4MIewq67YAgJm1BrAXwPMhhJ8kXvsrgM9D\nCH9N/J/mmSGE+zOofX8C8JW3wGgptK0mgJoFF0AF0AVAH2TA+SPt64ZSOH+lcYW/BMDaEEJuCOEw\ngJcA8JKpZUPGLI4ZQpgDYHehl28AcGz62Bgc/ZCUCSnaB2TAOQwhbAshLEo83gvg2AKoGXH+SPuA\nUjh/pRH42gA2FXj+Gf77D8wUAoBpZrbAzPi81rKjRgghL/E4DwCvsVw2ZNQCowUWQP0IGXj+ymKB\n1tII/A+h3++KEEIrAB0A3Ju4Zc1YwtHvYZl2XjNqgdHE7fJrOLoA6lcFXSacv7JaoLU0Ar8ZQN0C\nz+vi6FU+Yzi2Tl4IYQeACTj6NSTTyEt8/4OZZQPgK12UMiGE7SEBgOdQhuewwAKoLxxbABUZdP5S\nLdBaGuevNAK/AEAjM2tgZicDuAUAL2dTiphZRTM7PfG4EoBrkWJxzDJmEoDeice9AUwk25Y6iRAd\nI+UCo6XQjqQLoCJDzh9boLXAZmk7f6Uy0s7MOgAYBiALwMgQwsNpf9MiYmbn4OhVHTi69FZOWbfP\nzMYCaAOgGo5+3/w/AK8DeBlAPQC5ALqFEL7IkPb9CUBbHL0dzV9gtMB35tJs25UA3gOwBP+9bf89\ngPnIgPOXon1/ANAdpXD+NLRWiIjQSDshIkKBFyIiFHghIkKBFyIiFHghIkKBFyIiFHghIkKBFyIi\n/h9iUgKJrb1W7QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10cc4b438>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPwAAAD8CAYAAABTq8lnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGMlJREFUeJzt3XtwlOXZBvDrNnIKp3BKOIWDHCwIlFQQMTjgAZSx2uJY\nKSpFRjut1n6MdlprZ9SOX1sdO7XW2rH9RhStTLUVQasVOWgR1HDQhOjH+XwKSQTkqBbx+f5gSUPc\nve6YZJPt91y/GcfdvfbdfbPJzbu7z/s8t4UQICJxOKOpd0BEGo8KXiQiKniRiKjgRSKigheJiApe\nJCJ1Lngzu9zM1pnZRjO7syF3SkTSw+oyDm9mWQDWA7gUwG4AKwFMCSGsrXYfDfCLNKEQgtW87cw6\nPtZ5ADaFELYBgJk9C+AbANZWv1PHjh2rLh87dgzZ2dlV16dMmUKf4Oyzz6b5+++/T/PNmzfTfNy4\ncaddf/3113HxxRdXXT948CDdvk+fPjSfNWsWzW+66Saal5aWnnZ95cqVGDlyZNX1AwcO0O179epF\nc+/nW7x4Mc0BYMyYMVWXS0pKMHz48NPyr3zlK3T7du3a0XzZsmU07927N83Lysro/nmP7/2OWrVq\nRfOcnByaP/zww1WX9+7di65du56We39jhYWFKbOf/OQnSW+v61v6HgB2Vru+K3GbiGSwuha83q6L\n/Aeq61v63QDyq13Px8mj/GmOHTtWddnsCx8nMkrfvn2beheo7t27N/UuUDXfjmaaTN+/Nm3a1Gv7\nzZs3ux9jgboX/CoAA8ysD4A9ACYD+MKH8uqf2TNdphd8jx6Z/Ykp0wsq0/evvgXfr18/9OvXr+r6\nokWLkt6vTgUfQvjMzG4D8BqALAAzq39DLyKZqU7DcrV6YLMwffr0lPlZZ51Ft6/+DWsylZWVddqv\nU7xvWPv370/zw4cP09z7hvpXv/oVzX/0ox/RvLi4mObeKMfu3btp/umnn9IcAHJzc2nu/Y4WLFhA\nc/b3U5vtR4wYQfOSkhKajx07lubz5s2jefUjbjLeSMqhQ4dovmfPnpTZyy+/nHRYTmfaiUREBS8S\nERW8SERU8CIRUcGLREQFLxIRFbxIROp6pl2tDBo0KGW2adMmum3N2WI1nX/++TT3xpnLy8tpvnTp\nUppPnjyZ5uvXr6f5jBkzaO6No3unKjdr1ozm7du3p/l7771HcwA44wx+vHjllVdofskll9D8s88+\no7k3zu79DQwYMIDm7O8XAJo3b05zb0ZiRUUFzW+//Xaa/+53v6N5MjrCi0REBS8SERW8SERU8CIR\nUcGLREQFLxIRFbxIRNI6Dv/cc8+lzKZNm0a33b9/P83z8vJovmXLFpp7K9yMHz+e5rt2fWFFr9Ow\nucoAsGTJEppv27aN5jVX3a3JG4fv1q0bzb1x+to8x3XXXUfzdevW0dxbtfXEiRM0r77KbzLVV41N\nxhtn99Y8OHLkCM0HDhxI81/+8pc0HzZsGM2T0RFeJCIqeJGIqOBFIqKCF4mICl4kIip4kYio4EUi\nktZxeLa2+sKFC+m2d9xxB80feeQRmlfvXJuMNxf5nXfeofnEiRPrlR8/fpzmc+fOpXnPnj1p7q2r\nv337dpq3bNmS5gBw9OhRmnsdbr1zIbz57J988gnNvXHya665hubefHZvPr3X4dhb9957/hUrVtA8\nGR3hRSKigheJiApeJCIqeJGIqOBFIqKCF4mICl4kIvUahzezbQAOATgB4HgI4bzq+R//+MeU23rj\nwIWFhTT3xiiHDBlCc28ce8eOHTT35uvPnz+f5p07d6b56NGjad6hQweae73HvTF0b758bfTo0YPm\nL7zwAs29/uqDBw+m+Zw5c2jurWvvrf0/a9Ysmnu/A2/d/Y0bN9I8KyuL5snU98SbAGBcCIH/9YtI\nRmiIt/T8n0ERyRj1LfgAYJGZrTKz7zbEDolI+tT3LX1hCKHMzLoAWGhm60IIVU3Zqq/LlpOTg5yc\nnHo+nYgkU1FRgcrKSvd+9Sr4EEJZ4v+VZjYXwHkAqgreW4RQRBpGbm4ucnNzq66vWbMm6f3q/Jbe\nzLLNrG3icmsAEwDw6UEi0qTqc4TPAzA3MXRxJoDZIYQFDbJXIpIWdS74EMJWAMPZfYYOHZoyu+WW\nW+jje2OM3lzqoqIimvfu3Zvm3rrwEyZMoDn72QHggw8+oHmnTp1o/vTTT9P8t7/9Lc29+fplZWU0\nB/x1273cW1f+nHPOoXlpaSnNvd+xd66Bl//zn/+kuefxxx+n+UUXXUTzkpKSL/2cOtNOJCIqeJGI\nqOBFIqKCF4mICl4kIip4kYio4EUiktZ16dna7zNnzqTbXn311TRPdergKcOH01MEsHLlSprPmDGD\n5t5c6dWrV9Pc4/UWv//++2nujaO3atWK5t56AYA/DuyNY3v92b1zCT7++GOat2jRguaDBg2i+a23\n3krznTt30tx7Ddu2bUtzb137/Px8miejI7xIRFTwIhFRwYtERAUvEhEVvEhEVPAiEVHBi0QkrePw\njDeffP369TT31nW/4IILaH7o0CGaHz58mObeuvVe7/JRo0bVK/fWLPfGmDds2EDz7t270xzw143f\ntWsXzb01B7797W/T3Bun9n4Gbz66t6bBt771LZp7S7w9+OCDNB85ciTNvd4EyegILxIRFbxIRFTw\nIhFRwYtERAUvEhEVvEhEVPAiEUnrODybj+z1Tz/jDP5vkbcu/YIFvCdG69ataZ6dnU1zby621xvd\n+/m8den37NlD8xdffJHm48ePp/nzzz9Pc8Bfc8CbD+7N2ffOtSguLqb5Rx99RPP9+3mX86lTp9J8\n6dKlNN+8eTPNb7zxRpp7v0PvXJJkdIQXiYgKXiQiKniRiKjgRSKigheJiApeJCIqeJGIWAiB38Hs\nCQBXAKgIIQxN3NYRwHMAegPYBuDaEMJHNbYL119/fcrHzc3Npc/76quv0vziiy+muTfO7s03r6ys\npHnXrl1p/uGHH9Lc632+detWmntzwb0x8GbNmtVre8AfZ+7fvz/NO3bs6D4H89lnn9F89uzZNL/j\njjto/s4779D8D3/4A83btWtH88suu4zm3noCkyZNSpmNHTsWIYQvNE+ozRH+SQCX17jtpwAWhhAG\nAlicuC4iGc4t+BDCUgAHatx8FYCnEpefAvDNBt4vEUmDun6GzwshlCculwPIa6D9EZE0qve59CGE\nYGZJvwgoLS2tupyXl4e8PP27IJIOxcXFbq8/oO4FX25mXUMIe82sG4CkXSOHDRtWx4cXkS+joKAA\nBQUFVddnzZqV9H51fUv/EoBpicvTAMyr4+OISCNyC97M/gLgbQBnm9lOM5sO4AEA481sA4CLE9dF\nJMO54/B1fmCzcNttt6XMc3Jy6PbeOPa+ffto7o1xrlq1iuZ79+6l+fe//32ae3OlR4wYQfMLL7yQ\n5t4Yce/evWnuzacvKiqiOeDv46ZNm2h+6aWX0nzLli00N/vCMPNpvHMVvDUPvO3PP/98mi9ZsoTm\nBw7UHPw6nXeeAfsbLSoqqvM4vIj8P6GCF4mICl4kIip4kYio4EUiooIXiYgKXiQiaR2HZ/29vfng\n69ato7l3Xn6XLl1o7s3lbt++Pc0rKpKeTVzFG2P+/PPPaV59HkIy48aNo3l5eTnNvdOevZ8fAJYt\nW0Zzrz/6sWPHaL59+3aajx49muZHjx6l+RtvvEFzr399r169aO69ht66+l5tsjUZHnroIY3Di8RO\nBS8SERW8SERU8CIRUcGLREQFLxIRFbxIRNLaH75t27Yps6ysLLqtN47uzYXesWMHzb0x1gkTJtD8\nxIkTNG/RogXNN2zYQHO25jgAHD9+nOYrVqyg+cGDB2men59PcwAYM2YMzb219YcMGUJzr3+71z/+\niSeeoPk111xDc28+/8qVK2nu/XzeuvnPPvsszdesWUPzZHSEF4mICl4kIip4kYio4EUiooIXiYgK\nXiQiKniRiKR1HJ6t2+31vvbW5C4rK6P5wIEDaX7jjTfS/NChQzT/17/+RfO33nqL5j169KC517+9\nefPmNP/FL35B83fffZfmrVq1ojkA7N69m+be7/CMM/jxpmXLljT31ubv0KEDzb3XePjw4TTv1q0b\nzVevXk3ze++9l+be6+utJ5CMjvAiEVHBi0REBS8SERW8SERU8CIRUcGLREQFLxIRd116M3sCwBUA\nKkIIQxO3/RzAzQAqE3e7K4Qwv8Z2gfVQz83Npc87atQomr/++us098aAvXFmr7/6VVddRXNvrrK3\n7rz38//+97+n+d13301zbz59bXz88cc099YkmDhxIs1feOEFmnvnOrRu3Zrm+/bto7nXW2DWrFn1\nevxbbrmF5qz/OwBccsklKbOxY8fWeV36JwFcXuO2AOChEEJB4r/5SbYTkQzjFnwIYSmAA0kivuSM\niGSc+nyG/6GZrTazmWaW02B7JCJpU9dz6R8DcF/i8n8D+A2Am2reqfqaX927d3fPHxeRuikuLkZJ\nSYl7vzoVfAihqpOimT0O4O/J7jdy5Mi6PLyIfEkFBQUoKCioup7qC8U6vaU3s+rThCYBeL8ujyMi\njcs9wpvZXwCMBdDZzHYCuBfAODMbjpPf1m8F8L207qWINIi09oefPn16ytxbk9vrLV5YWEhzb5zf\n+7m9/vHeXOihQ4fSfMuWLTT3fv7s7Gya9+/fn+YXXHABzZcvX05zwF/b3+vf7v0OnnzySZp7/df/\n8Y9/0PxrX/sazb3f0VlnnUXzM8/kx9ONGzfSvF+/fjRnf6N/+9vf1B9eJHYqeJGIqOBFIqKCF4mI\nCl4kIip4kYio4EUiktZ16T/55JOUmXfarbcmuNff3BuH99aVv+uuu2j+3nvv0bxnz54098b5vXF2\nb938t99+m+bdu3eneceOHWkOAH379qX5n/70J5qPGDGC5l7/ee819nofHD58mOb1HWffunUrzb1x\ndm/uSW3Ona9JR3iRiKjgRSKigheJiApeJCIqeJGIqOBFIqKCF4lIWsfh2ViuN1+bjeEDQIsWLWju\nzWVeuHAhzb3e5Xv27KF5ly5daH7dddfR/OWXX6b59u3baV5ZWUnzK664gubl5eU0B/x148877zya\ne+PYO3fupPmmTZto7q1LP3XqVJo/9thjNM/KyqK5dy6D9zc4btw4ml900UUps/nzk68cryO8SERU\n8CIRUcGLREQFLxIRFbxIRFTwIhFRwYtEJK3j8GysuHPnznXeFgDy8/Np7s3V/upXv0rzSZMm0fz5\n55+nuTdO7/UWf/TRR2nurdnurdvvrTs/ZMgQmgP+OLrXP/6cc86huTffvaioiOZTpkyh+f33309z\nrzfA2LFjaf7AAw/Q3OtdUN/5/MnoCC8SERW8SERU8CIRUcGLREQFLxIRFbxIRFTwIhGh/eHNLB/A\n0wByAQQA/xNCeMTMOgJ4DkBvANsAXBtC+KjGtuHXv/51yscuLS2lO+bNVfbmYntjxN5c5t27d9Pc\nm+9+5ZVX0twbo169ejXNvbnS3hit11u9oKCA5oD/GnnnUnjz5b01Dby1971xfq+3gPc78noLeOvu\nf/rppzQvKyujOeut8Oijj9apP/xxALeHEM4BcD6AH5jZIAA/BbAwhDAQwOLEdRHJcLTgQwh7Qwgl\nictHAKwF0APAVQCeStztKQDfTOdOikjDqPVneDPrA6AAwHIAeSGEU2sglQPIa/A9E5EGV6tz6c2s\nDYA5AGaEEA6b/fujQQghmFnSLwIWLFhQdblfv35uLy0RqZtdu3a536kAtSh4M2uGk8X+5xDCvMTN\n5WbWNYSw18y6AahItu2ECRO+xC6LSF317NnztMlGK1euTHo/+pbeTh7KZwJYE0J4uFr0EoBpicvT\nAMyrua2IZB7vCF8I4AYApWZWnLjtLgAPAPirmd2ExLBc2vZQRBoMLfgQwjKkfhdwqffg69evT5md\nffbZdNsbbriB5rfeeivNvTHUI0eO0NybL3/ttfzfOG+MuGvXrjQfNmxYvR7/61//Os29NdGPHj1K\nc8DvHeBZunQpzSsqkn5SrOKtOeD1d/e298bRPd7jZ2dn09w7l8I7VyMZnWknEhEVvEhEVPAiEVHB\ni0REBS8SERW8SERU8CIRSeu69Gzt+X379tFtO3XqRPPPP/+8XnlJSQnNL7vsMprPm8dPLjx27BjN\nZ8+eTfMxY8bQ/PLLL6d5y5Ytae6dx1CbcfhFixbRnK21APjj1Bs2bKD5gAEDaP7mm2/SfO/evTRf\nu3Ytzb2/oe7du9PcW5Ng3bp1NJ88eXLKbMWKFUlv1xFeJCIqeJGIqOBFIqKCF4mICl4kIip4kYio\n4EUiktZx+P79+6fMvP7w3hjloUOHaO7Ndx84cCDNT5w4QfNt27bRvHXr1jSfMWMGzcvLy2nu9Q73\nxtG98wTy8vx1Sb1x7B07dtDcW3f+5ptvpvncuXNp3qxZM5q3adOG5i1atKC519/d+/mXL19Oc2/N\nhVWrVtE8GR3hRSKigheJiApeJCIqeJGIqOBFIqKCF4mICl4kImkdh1+zZk3KzFszfP/+/TTv0KED\nzb0+W4MGDaJ59bY9ybRq1Yrm3hiztya79/znnnsuzRcvXkzzVPOlTxk9ejTNAeD48eM0nzRpEs29\n3gEffPABzb1ehd7vYNmyZTRn880B/1yHK6+8kubefH2vf31WVhbNk9ERXiQiKniRiKjgRSKigheJ\niApeJCIqeJGI0II3s3wze8PM/tfMPjCz/0rc/nMz22VmxYn/+JrJIpIRjK0dbmZdAXQNIZSYWRsA\n7wL4JoBrARwOITxEtg0//vGPUz62N1fZ633dp08fmnvz3UtLS2nujRF37NiR5m+99RbNr776app7\nY9DeuvveOLrXF8BbTwAAzjyTn8axZMkSmnvnGnjjzN44vNe//uDBgzQfPnw4zdu1a1ev5/fm47/y\nyis0Z+c5TJkyBSEEq3k7/Y2FEPYC2Ju4fMTM1gLokYi/8GAiktlq/RnezPoAKABQlLjph2a22sxm\nmllOGvZNRBpYrQo+8Xb+eQAzQghHADwGoC+A4QDKAPwmbXsoIg3GPZfezJoBmAPgmRDCPAAIIVRU\nyx8H8Pdk21b/HJufn49evXrVd39FJIk1a9bQuSun0II3MwMwE8CaEMLD1W7vFkIoS1ydBOD9ZNsX\nFhbWeodFpO4GDx6MwYMHV12fM2dO0vt5R/hCADcAKDWz4sRtPwMwxcyGAwgAtgL4Xn13WETSz/uW\nfhmSf85/NT27IyLplNb58Gzt9N69e9NtP/zwQ5pff/31NPd6k2dnZ9Pcm+vsjcN78+V37txJ8/Hj\nx9PcG8N99913ae6tu1+bcfguXbrQ/Dvf+Q7Nvfnew4YNo7nXP/2ll16iubf2v9e/PieHD05NnTqV\n5vfddx/NvXXv58+fT/NkdGqtSERU8CIRUcGLREQFLxIRFbxIRFTwIhFRwYtEJK3j8Gwsu6ioKGUG\nAPfccw/Nvf7sr732Gs23b99O87vvvpvm3lztk2clp+bNtX7mmWdoPmrUKJrXd11/7zwDAGjevDnN\nO3XqRHOvB713LoY3jn7nnXfS3FvzwJsv763pUFxcTPPOnTvTvH379jR/8cUXaZ6MjvAiEVHBi0RE\nBS8SkUYreO/zWFOrzbnjTamysrKpd4HyvlNpaps2bWrqXaC8NQYbigo+QQVfPyr4+vG+ZG0oeksv\nEhEVvEhE6Lr09Xpgs/Q8sIjUSrJ16dNW8CKSefSWXiQiKniRiDRKwZvZ5Wa2zsw2mhk/wbkJmNk2\nMytNNMZckQH784SZlZvZ+9Vu62hmC81sg5ktaMpuPyn2LyMajJIGqBnx+jV1g9a0f4Y3sywA6wFc\nCmA3gJUApoQQ1qb1ib8EM9sK4NwQQuMMhjrM7EIARwA8HUIYmrjtQQAfhhAeTPyj2SGE8NMM2r97\n4TQYbaR9S9UAdToy4PWrT4PWhtAYR/jzAGwKIWwLIRwH8CyAbzTC835ZGdMcM4SwFMCBGjdfBeCp\nxOWncPKPpEmk2D8gA17DEMLeEEJJ4vIRAKcaoGbE60f2D2iE168xCr4HgOprMu/Cv3/ATBEALDKz\nVWb23abemRTyQgin5oOWA+BzS5tGRjUYrdYAdTky8PVrigatjVHw/wnjfoUhhAIAEwH8IPGWNWOF\nk5/DMu11zagGo4m3y3NwsgHq4epZJrx+TdWgtTEKfjeA/GrX83HyKJ8xTvXJCyFUApiLkx9DMk15\n4vMfzKwbgArn/o0qhFAREgA8jiZ8Das1QP3zqQaoyKDXL1WD1sZ4/Rqj4FcBGGBmfcysOYDJAHhL\nkEZkZtlm1jZxuTWACUjRHLOJvQRgWuLyNADzyH0bXaKITknZYLQR9iNpA1RkyOvHGrRWu1vaXr9G\nOdPOzCYCeBhAFoCZIYT70/6ktWRmfXHyqA6cXPJrdlPvn5n9BcBYAJ1x8vPmPQBeBPBXAL0AbANw\nbQjhowzZv3sBjMPJt6NVDUarfWZuzH0bA+BNAKX499v2uwCsQAa8fin272cApqARXj+dWisSEZ1p\nJxIRFbxIRFTwIhFRwYtERAUvEhEVvEhEVPAiEVHBi0Tk/wCGGd5NHSJrkQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10c06e4a8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPwAAAD8CAYAAABTq8lnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGKlJREFUeJzt3XmQVOW5BvDnFUHZBZF9GBABBYfNsF/AwpGARNyixIix\njAZTcqNl8odgVW5SWsmNKRfKSpWliaEIlyCWuJBEFBAXiAiDQUFANhkYlplBWWRQYMTv/jHNZGbo\nft6xh57p1Pf8qiy7++nTffpMv5zu/jYLIUBE4nBOQ++AiNQfFbxIRFTwIhFRwYtERAUvEhEVvEhE\n0i54M5tgZp+Y2TYze/Bs7pSIZIal0w5vZo0AbAGQD2AvgAIAt4YQNle5jxr4RRpQCMFq3nZumo81\nFMD2EEIhAJjZ8wCuA7C56p1uvvnmyssbN25Ev379Kq8PGzaMPkGzZs1o3rdvX5q3bt2a5l9//XW1\n688++yymTZtW7TozatQomg8ePJjm69evp/mXX35Z7fqiRYswefLkyuvl5eV0+08//ZTm33zzDc3b\ntm1LcwCoerJYvnw5xo0bVy0/fPgw3b5FixY0Hz16NM0///xzmo8cObLy8hNPPIGf//zn1fJXX32V\nbv/VV1/R3DuGZmfUWzVV3+OvvfYarrnmmmp5Xl4e3f6Pf/xjymzevHlJb0/3I30XAEVVru9J3CYi\nWSzdgtfHdZH/QOl+pN8LIKfK9RxUnOWr2bhxY+Xlxo0bp/lU9eOKK65o6F2g+vTp09C7QPXo0aOh\nd4EaMWJEQ+8C1atXrzptX1JSgpKSEvd+6Rb8WgC9zKw7gH0ApgC4teadqn5nz3Yq+LpRwddNXQu+\nQ4cO6NChQ+X1DRs2JL1fWgUfQvjazP4bwBsAGgF4ruov9CKSndI9wyOEsBjAYnYf9ku19yv67t27\nae79Cn3vvffSPNW/gKc1atSI5p4tW7bQ/LzzzqN5165daf7ZZ5/RvHv37jQ/duwYzdeuXUtzADjn\nHP4T0PDhw2nu/Q2/+OILmvfs2ZPmDz/8cJ0e/5FHHqH53LlzaV5QUEDzu+66i+Z79pzxLbmaqmf0\n2lJPO5GIqOBFIqKCF4mICl4kIip4kYio4EUiooIXiUhaw2Nr9cBm4e67706ZDxw4kG5/9OjROj3/\nyZMnab5r1y6aT5w4keZeG25OTg7NDx48SPPly5fT3GuHv/POO2nujUY8fvw4zQFg82be18rrfdeq\nVSuaN2nShOZbt26luTdaraioiObjx4+n+ezZs2nu9bXwjo/XT+HGG29MmeXn5ycdHqszvEhEVPAi\nEVHBi0REBS8SERW8SERU8CIRUcGLRCTt8fC1cerUqZRZy5Yt6bYXXnghzb3x8l47fH5+Ps29NlKv\njXjhwoU0v+2222jujaf3xoKfjVlpPTVn/q3p0KFDNF+5ciXNO3fuTHNvlphOnTrRvLS0lOb//Oc/\nad6tWzear1ixgubTp0+n+bp162j+1ltv0TwZneFFIqKCF4mICl4kIip4kYio4EUiooIXiYgKXiQi\nGW2HLy4uTpl5y+J88sknNL/kkktoXlZWRnOvjffIkSM099q5vfH+3uvfuXMnze+55x6aP/DAAzSv\nudJrTbfffjvNAeCll16i+U033UTzqVOn0rx9+/Y0f+aZZ2g+YMAAmu/YsYPm999/P829uf1nzZpF\n86VLl9J827ZtNE9n7QSd4UUiooIXiYgKXiQiKniRiKjgRSKigheJiApeJCJ1aoc3s0IAXwA4BaA8\nhDC0as7GbHtrX3tjnU+cOEHzKVOm0HzTpk009+Y898bLe+u7e+PlhwwZQnNvTvVhw4bR3JsX35tT\nHQD69etH87y8PJp748298ere8y9YsIDm3toHy5Yto7n3Hrz33ntp7q09MHr0aJqns6ZEXTveBABX\nhhD4u0dEssLZ+EjPl/cQkaxR14IPAJaZ2Voz+8nZ2CERyZy6fqQfFULYb2YXAVhqZp+EECon8lq9\nenXlHbt06eJ+rxWR9OzYscMdGwDUseBDCPsT/z9gZi8DGAqgsuC9H45E5Ozo2bNntR/JUw3MSfsj\nvZk1M7OWicvNAYwHsCHdxxORzKvLGb4DgJcTS/KeC2BeCGHJWdkrEcmIjK4Pz+YdLygooNt36dKF\n5t687d54de/7jjfeft++fTRv3bo1zb1597399/opfPjhhzT31k731rcHgEmTJtH83HP5+eTw4cM0\n98Z7v/jiizT3/kbt2rWj+d69e2nuzbngPf8vfvELmnvz6rO+HL/5zW+0PrxI7FTwIhFRwYtERAUv\nEhEVvEhEVPAiEVHBi0Qko+3w//jHP1LmXju6t7b40KFDad6mTRual5eX03zNmjU097oN79+/n+aL\nFy+mef/+/Wnep08fmnvz+ntjwVu0aEFzALjiiito3rt3b5p77eDPPvsszfv27UvzDz74gObNmjWr\nUz579myajxw5kua5ubk09/qCsH4KV199tdrhRWKngheJiApeJCIqeJGIqOBFIqKCF4mICl4kIhld\nH57N7T548GC67fnnn0/zv/71rzT3xtMPHz6c5t26daO5t3a5N97fG+vstTF7a5N77fTe+vHe+vMA\n0Lx5c5p7+3jkyBGaX3755TTfvn07zb2/4YwZM2j+ve99j+beeP1p06bR3Fv7YNWqVTRfu3YtzZPR\nGV4kIip4kYio4EUiooIXiYgKXiQiKniRiKjgRSKS0fHwv/3tb1Pm3nhyb153b6yy10b66quv0txb\nm9xb+3vnzp00X7duHc29dvh//etfNC8uLqb5ddddR/NFixbRHAAuvfRSmnt/Y28N+kcffZTm3nj6\ngwf5Kubee6hx48Y09+Ys8NaP9/oRjB07luZNmjRJmU2aNEnj4UVip4IXiYgKXiQiKniRiKjgRSKi\ngheJiApeJCLueHgz+zOASQBKQwh5idvaAlgAIBdAIYBbQghnLPbN2hm98doff/wxzb3x8t5YaO/x\nBwwYQHNv7XNv/XmvDdmbl99be9xbe91rZ/f6AQDAVVddRXNvjfmNGzfS3GvHf+mll2juzZvvzZmw\na9cumg8aNIjm3tz/3nvwrbfeorlXQ8nU5gw/G8CEGrfNALA0hNAbwJuJ6yKS5dyCDyGsAFDzdDMZ\nwJzE5TkArj/L+yUiGZDud/gOIYSSxOUSAB3O0v6ISAbVeU67EEIws6Qd8qv29+7UqZM7j5uIpGfr\n1q3Ytm2be790C77EzDqGEIrNrBOApCNNvIkqReTs6N27d7XFO1977bWk90v3I/0iAHckLt8B4JU0\nH0dE6pFb8GY2H8B7APqYWZGZ3QngdwCuNrOtAMYlrotIlnM/0ocQbk0R5XvbsrbywsJCum3Pnj1p\n7n1f8cYy9+rVi+ZeO/vChQtp3rJlS5qvWLGC5t58AF4bs9cPoEePHjSvTTu8147u/Wazd+9emntz\nFniP37RpU5pfdNFFNPeOkTenwdChQ2nutaN778GuXbvSPBn1tBOJiApeJCIqeJGIqOBFIqKCF4mI\nCl4kIip4kYhkdH141s5YUlKSMgP8sdQTJtQcsVvd7t27aT5mzBiab9iwgeZeG2mrVq1o7nU73r9/\nP83nz59P82XLltHcayN+8sknaQ74fQXKy8vrtL3X18Jrx67r2gVlZWU0b9OmDc29eee9x/fmA/DG\n2yejM7xIRFTwIhFRwYtERAUvEhEVvEhEVPAiEVHBi0Qko+3wBw4cSJmNGDGCbvvNN9/Q/OTJkzT3\n2rG99durTheUjDfvu9fO/cYbb9C8bdu2NB8/fjzNr7zySpp7Y73Nzlha/AxeO7Y3J8GcOXNo/vDD\nD9P8+uv5ZMneeHVvPPyaNWto7rWTb926leZHjhyhudcXZe7cuTRPRmd4kYio4EUiooIXiYgKXiQi\nKniRiKjgRSKigheJSEbb4YcMGZIyW7VqFd32xIkTNPfWJvfGUjdp0oTmqZbqOW348OE0P3bsGM37\n9+9P86rr8iXTvHlzmntzvntt0F4/AwCYOHEizWfOnElzb+577zXcd999NJ86dSrNi4uLae61s69e\nvZrm3pwJl156Kc337NlD89LSpCu8UTrDi0REBS8SERW8SERU8CIRUcGLREQFLxIRFbxIRNx2eDP7\nM4BJAEpDCHmJ234N4G4Apwe8zwwhvF5zW9ZO6Y31feedd2jerl07mg8aNIjmBQUFNM/Pz6e5tza6\nN17/9dfPOFzVdOvWjeb79u2jeV5eHs294++NdQeAoqIimnvztm/atInm3pj/7t2707xFixY0f/nl\nl2nevn17mnvt5JdddhnNp0yZQvP33nuP5rXpK1FTbc7wswHUXPUhAHgihDAo8R9/94pIVnALPoSw\nAsChJJE/JYqIZJW6fIf/mZl9ZGbPmdkFZ22PRCRj0u1L/zSA0xOOPQLgcQB31bzTu+++W3k5NzcX\nubm5aT6diDClpaV0DsnT0ir4EEJlr30z+xOAvyW7n7dgo4icHe3bt6/2I+PmzZuT3i+tj/Rm1qnK\n1RsA8KVWRSQr1KZZbj6AsQDamVkRgF8BuNLMBqLi1/qdAO7J6F6KyFlhIYTMPLBZYO2IK1eupNv3\n6tWL5u+//z7Nvd8LvDZab/9uueUWmntro3vj7UePHk3zwsJCmn/11Vc0nzChZktrdSUlJTQHgPXr\n19P80KFkjTv/5vUl8MaLe6/ho48+ork3937Lli1p7q2d4K0t4LXje/0AWF+QmTNnIoRwxgtUTzuR\niKjgRSKigheJiApeJCIqeJGIqOBFIqKCF4lIRuelZ2uge22YHTt2pPkll1xC888//5zmVfv5J9Ol\nSxeae2OV3377bZpfe+21NF+6dCnNR40aRXNvzveysjKae68fAJYsWUJzrx3dW3/dm3v/D3/4A817\n9+5Nc6+vQuvWrWnurU3gzZng9aXw+jF4cxokozO8SERU8CIRUcGLREQFLxIRFbxIRFTwIhFRwYtE\nJKPt8Kyt3Rsv7rWTN27cmOY33HADzb05zb021B/84Ac099rpN2zgkwRdfvnlNPfGUnvz4nuvf968\neTQHgHPO4eeLLVu20NxbP/2mm26iufce2r17N829uf+98fIHDx6kuddO7q2t4P0NvfdQMjrDi0RE\nBS8SERW8SERU8CIRUcGLREQFLxIRFbxIRDLaDs/Wx961axfd1hvP/sUXX9C8tLSU5sOGDaN5586d\naT5//nya/+hHP6K510b8wx/+sE7be8f3yJEjdcoBYODAgTS/8MILae7NK//000/T/Mknn6S59x7p\n0KEDzb1jcOrUKZq3atWK5ps2baK5tzaANydEMjrDi0REBS8SERW8SERU8CIRUcGLREQFLxIRFbxI\nRGg7vJnlAPgLgPYAAoBnQwhPmVlbAAsA5AIoBHBLCOFwze03b96c8rG9scBjxoyh+eLFi2nuzUnu\nGTx4MM3ZawP88fDeeH2vjdcbr3/xxRfT3OtH4PVTAPx2bm/MfrNmzWj+y1/+kubPPPMMzb1+Al5f\nhc8++4zmEydOpHlBQQHNmzZtSvPx48fTnK37kIp3hi8H8EAIoR+A4QCmm9llAGYAWBpC6A3gzcR1\nEclytOBDCMUhhA8Tl8sAbAbQBcBkAHMSd5sD4PpM7qSInB21/g5vZt0BDAKwGkCHEMLpfn8lAHgf\nRRHJCrXqS29mLQAsBHB/COFo1bm+QgjBzEKy7d55553Ky7m5ue48aiKSnqKiIvc3E6AWBW9mjVFR\n7HNDCK8kbi4xs44hhGIz6wQg6UiVsWPHfotdFpF05eTkVJs08/333096P/qR3ipO5c8B2BRCmFUl\nWgTgjsTlOwC8UnNbEck+3hl+FICpANab2brEbTMB/A7AC2Z2FxLNchnbQxE5a2jBhxBWIvWngHzv\nwdl447y8PLrt+eefT/Nx48bRfNmyZTT/8Y9/TPNXXuEfWrx549mc/ACwcuVKmnfq1Inm3lhur412\n8uTJNN+xYwfNAf8r25tvvklzb7z7o48+WqfH99Z/HzJkCM379u1L8wULFtB8586dNN+2bRvNf/rT\nn9I8nd/E1NNOJCIqeJGIqOBFIqKCF4mICl4kIip4kYio4EUiktF56dm82147tre2+Pbt22l+1VVX\n0fzBBx+keVlZGc29Ode9NlxvLPnbb79N8x49etD8+PHjNA8h6fCHSt/97ndpDgALFy6kube++Y03\n3kjzw4fPmGKhmscff5zmixYtornXTn7eeefRvF+/fjTv2LEjzb129q+//prmK1asoHkyOsOLREQF\nLxIRFbxIRFTwIhFRwYtERAUvEhEVvEhEMtoOz9qqvTbEffv20fzcc/mue3OCe/Oue3OaX3DBBTT/\n9NNPae614584cYLm3lhqb/14b7x+1fkIU/H6Knjt3N6Yf6+vhfce8OZU8P4GXjt6mzZtaF5cXEzz\nVNNQnXbttdfS3Js3Pxmd4UUiooIXiYgKXiQiKniRiKjgRSKigheJiApeJCLmjYtO+4HNwpw5c1Lm\nhYWFdHuvnXjEiBE09+Ykz83NpfkLL7xAc6+Nt127djT3Xr/XRvzll1/W6fnPOYf/W+/1AwD8dnSv\nr4KndevWNF+3bh3NR44cSXPvNb744os0v+aaa2j+/PPP0/z73/8+zTds2EDzU6dOpcxmzJiBEILV\nvF1neJGIqOBFIqKCF4mICl4kIip4kYio4EUiQgvezHLM7C0z22hmH5vZfYnbf21me8xsXeK/CfWz\nuyJSF954+HIAD4QQPjSzFgA+MLOlAAKAJ0IIT7CN2Rrm3njuoUOH0txbX90bS33zzTfTvEuXLjRv\n0aIFzY8ePUpzbyz3sWPHaN6oUSOad+7cmeYbN26kedeuXWlem8coKiqieX5+Ps29edm9uf9nzZpF\n84svvpjmbdu2pbn3+ry+IMuXL6e510/Aew8l3YaFIYRiAMWJy2VmthnA6Uo4o1FfRLJbrb/Dm1l3\nAIMAnJ6m42dm9pGZPWdmdetSJSL1olYFn/g4/yKA+0MIZQCeBtADwEAA+wHwNX9EJCu4XwLMrDGA\nhQD+L4TwCgCEEEqr5H8C8Ldk286dO7fycv/+/TFgwIC67q+IJFFUVOT+pgA4BW9mBuA5AJtCCLOq\n3N4phLA/cfUGAEl7+d9+++213mERSV9OTg5ycnIqr69atSrp/bwz/CgAUwGsN7PTQ5MeAnCrmQ1E\nxa/1OwHcU9cdFpHM836lX4nk3/MXZ2Z3RCSTMjoe/qGHHkqZ33bbbXR7b+1xr53bG+/ufd+ZNm0a\nzceNG0fzp556iubl5eU079OnD80rvm2lVlJSUqftvXZ+wO9L4b1Gb953bzz98ePHae6t775kyRKa\nHzhwgObevPTenA7e39jri8Hmre/Vq5fGw4vETgUvEhEVvEhEVPAiEVHBi0REBS8SERW8SEQyuj48\nawt+7LHH6LZeO7rXf8Cbk9xro/XWNvfacLdu3Upzb071Q4cO0bxp06Y09/bfm1P973//O80B/zVO\nnz6d5nv37qW5N97em7PAm9v/5MmTNP/Od75D8zFjxtD89ddfp/nHH39M87r2I0hGZ3iRiKjgRSKi\ngheJSL0V/P79+/07NaA1a9Y09C5QW7ZsaehdoLy++w3tgw8+aOhdoPbs2VMvz6OCTygoKGjoXaBU\n8HWT7QXv/YB5tugjvUhEVPAiEcnoePiMPLCI1Eqy8fAZK3gRyT76SC8SERW8SETqpeDNbIKZfWJm\n28zswfp4zm/DzArNbH1iYcwGb5A3sz+bWYmZbahyW1szW2pmW81sSUOu9pNi/7JigVGyAGpWHL+G\nXqA149/hzawRgC0A8gHsBVAA4NYQwuaMPvG3YGY7AVwRQjjY0PsCAGY2GkAZgL+EEPISt/0ewGch\nhN8n/tFsE0KYkUX79ysAR70FRuth3zoC6Fh1AVQA1wO4E1lw/Mj+3YJ6OH71cYYfCmB7CKEwhFAO\n4HkA19XD835bWbM4ZghhBYCaw+UmA5iTuDwHFW+SBpFi/4AsOIYhhOIQwoeJy2UATi+AmhXHj+wf\nUA/Hrz4KvguAqnNC78G/X2C2CACWmdlaM/tJQ+9MCh1CCKe7s5UASL0Wd8PJqgVGqyyAuhpZePwa\nYoHW+ij4/4R2v1EhhEEAJgKYnvjImrVCxfewbDuuWbXAaOLj8kJULIBabRGDbDh+DbVAa30U/F4A\nOVWu56DiLJ81Tq+TF0I4AOBlVHwNyTYlie9/MLNOAEqd+9erEEJpSADwJzTgMayyAOrc0wugIouO\nX6oFWuvj+NVHwa8F0MvMuptZEwBTACyqh+etFTNrZmYtE5ebAxiPFItjNrBFAO5IXL4DwCvkvvUu\nUUSnpVxgtB72I+kCqMiS48cWaK1yt4wdv3rpaWdmEwHMAtAIwHMhhP/N+JPWkpn1QMVZHaiY8mte\nQ++fmc0HMBZAO1R83/wfAK8CeAFANwCFAG4JIRzOkv37FYArUfFxtHKB0Srfmetz3/4LwLsA1uPf\nH9tnAliDLDh+KfbvIQC3oh6On7rWikREPe1EIqKCF4mICl4kIip4kYio4EUiooIXiYgKXiQiKniR\niPw/ieLfWdAduIsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10cad8780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    plt.figure()\n",
    "    plt.imshow(mlp.weights_[0].get_value()[:, i].reshape(28, 28), cmap=plt.cm.Greys,\n",
    "           interpolation='nearest');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 7.74 ms, sys: 206 µs, total: 7.94 ms\n",
      "Wall time: 5.24 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[(10, 1000), (10, 1000), (10, 1000), (10, 10)]"
      ]
     },
     "execution_count": 263,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "[a.shape for a in mlp.forward_shuffle(X_train[:10], layer_idx=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "sims = cosine_similarity(out[:100], out_shuffled_activations[:100]).ravel()\n",
    "inner_sims = cosine_similarity(out[:100]).ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEACAYAAAC6d6FnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X90FdW99/H3l2gQbPghaBAC4o+ooPzwotArF4l9qDe6\nFK26QFRKW+pylQrqtVV4elGxvVRt9fHXFR+9WqCAXrBKoVIFsQfR52oMBUQiQqQBE03UooBWMMj3\n+eNMwgGSnDn5cU5gPq+1stizZ8/MniGZ7+w9P7a5OyIiEj1tMl0BERHJDAUAEZGIUgAQEYkoBQAR\nkYhSABARiSgFABGRiEoaAMzsRjNbZ2bvmNmNQd4xZrbMzDaa2VIz65RQfoqZbTKzDWZ2QUL+oGA9\nm8zswZbZHRERCavBAGBmZwI/Bs4BBgAXm9nJwGRgmbufCiwPpjGzvsBooC9QCDxqZhasbgYw3t3z\ngXwzK2yB/RERkZCStQBOB950913u/g2wArgCGAnMCsrMAi4L0pcCT7t7tbuXAaXAEDM7Hshx96Kg\n3OyEZUREJAOSBYB3gGFBl0974CIgD8h196qgTBWQG6S7A+UJy5cDPerIrwjyRUQkQ45oaKa7bzCz\ne4ClwJfAGuCbA8q4mel7EiIih5gGAwCAuz8FPAVgZv9B/Eq+ysy6uXtl0L3zcVC8AuiZsHheUL4i\nSCfmV9S1PQUTEZHUubslL7W/ME8BHRf82wu4HJgHLALGBUXGAQuD9CLgKjPLNrMTgXygyN0rgR1m\nNiS4KTw2YZm6diTyP3fccUfG69BafnQsdCx0LBr+aaykLQDgWTPrAlQDE9x9u5ndDcw3s/FAGTAq\nOHGXmNl8oATYE5Svqd0EYCbQDlji7i82utYiItJkYbqAzqsjbxswop7y04HpdeSvAvo1oo4iItIC\n9CZwK1VQUJDpKrQaOhb76Fjso2PRdNaU/qOWYGbe2uokItKamRneiJvAYe4BiEgz2fdivEjjNOcF\nsgKASJqphSuN1dwXELoHICISUQoAIiIRpQAgIhJRCgAi0qBf//rXXHfddZmuBgArV67k9NNPr3d+\nWVkZbdq0Ye/evU3aTiwWo2fPnskL1uP1118nPz+fnJwcFi1aRFVVFeeddx4dOnTgZz/7GXfeeSdj\nx45tUh2bg24Ci2TQ3LmL2bmz5dafkwPXXHNJk9YxZcqUZqpN0w0bNowNGzbUTvfu3ZunnnqK73zn\nOxms1cFuv/12Jk2axMSJEwH45S9/yXHHHceOHTsAmDZtWiarV0sBQCSDdu6EHj2adoJuSEXF4hZb\nd2sQPP+e6WocZOvWrfTt27d2esuWLfTp06d2urXUWV1AIgLAPffcQ15eHh06dOD000/nlVdeAdiv\nu6Kmi2XmzJn06tWLLl268Nhjj/HWW2/Rv39/OnfuXHvVC1BaWsrw4cPp1KkTxx57LFdddVWd2x43\nbhz3338/ABUVFbRp04ZHH30UgPfff58uXboA+3fNjB07lq1bt3LJJZeQk5PDb3/729r1zZkzhxNO\nOIFjjz2W6dMP+jJNrSVLlnDGGWfQoUMH8vLyuO+++/abf//995Obm0v37t2ZOXNmbX5BQQFPPvlk\n7fTMmTMZNmwYACeffDKbN2+urdfVV1/N7Nmzuffee+nQoQPLly8/6HHON954g3PPPZfOnTszcOBA\nVqxYUW+dm5NaACLCe++9x3/+539SXFxMt27d2Lp1K3v27AHqfva8qKiI0tJSVqxYwcUXX8xFF13E\nK6+8wtdff81ZZ53FqFGjGDZsGFOnTqWwsJAVK1bw9ddfU1xcXOf2CwoKeP755/m3f/s3VqxYwUkn\nncSrr77KhAkTWLFiBeedd9Anyfj973/Pa6+9xpNPPlnbBVRWVgbE++A3btzIe++9x+DBg7n88svr\nvHcwfvx4nn32WYYOHcr27dvZvHlz7bzKykp27NjBhx9+yNKlS7nyyiv53ve+R8eOHTGzep/Jf//9\n9znxxBP3q1fbtm3p2bMnd911FxC/l1GjoqKCiy++mDlz5lBYWMjLL7/MFVdcwYYNG+jatWud22gu\nrbIFsHv37lA/Nb+gItI0WVlZ7N69m/Xr11NdXU2vXr046aSTgLq7K6ZOnUp2djbf/e53a69yu3bt\nSvfu3Rk2bBirV68GIDs7m7KyMioqKsjOzubcc8+tc/vnnXcer732Gu7OypUrufXWW3n99dcBWLFi\nBcOHD09pf+644w7atm1L//79GTBgAGvXrq2zXHZ2NuvXr2fHjh107NiRs846q3bekUceye23305W\nVhYXXngh3/rWt3jvvfdSqkei+rp95syZw0UXXURhYXyY9BEjRnD22WezZMmSRm8rrFbZAvjdwt8l\nLbP3m70M6z+MfmfqA6MiTXXKKafwwAMPcOedd7J+/Xr+9V//lfvvv5/jjz++zvK5ubm16Xbt2h00\nvTO4s33vvfcydepUBg8eTOfOnbnlllv44Q9/eND6Tj75ZI4++mjWrFnDypUrmTp1Kk8++SQbN27k\n1Vdf5aabbkppf7p161abbt++PV9++WWd5f7whz/wq1/9ismTJ9O/f3/uvvtuvv3tbwPQpUsX2rRp\ns996vvjii5TqEcaWLVtYsGABixfvu1+zZ8+etNzYbpUBoMfA5MMFV35QyV5v2qNeIrLPmDFjGDNm\nDDt37uT666/ntttuY/bs2Y1aV033SG5uLo8//jgQ75YZMWIEw4cPr21dJBo+fDgLFiygurqa7t27\nM3z4cGbOnMlnn33GwIEDG9xOY5199tksXLiQb775hocffphRo0axdevWpMsdffTR+wWVysrKRteh\nV69ejB07tvY4pVOr7AISkfTauHEjr7zyCrt376Zt27YcddRRZGVlNXp9Nd0dCxYsoLy8HIBOnTph\nZvtdVScaPnw4jzzySG1/f0FBAY888gjDhg2r90Sfm5vL+++/H7o+iaqrq5k7dy7bt28nKyuLnJyc\n0Ps8cOBAnnvuOb766itKS0v3uyEcdvs1rr32WhYvXszSpUv55ptv2LVrF7FYjIqKOkfNbVZJWwBm\nNgW4FtgLrAN+CBwN/DdwAsGIYO7+eUL5HxEfPH6Suy8N8gcRHxHsKOIjgt3YzPsicsjJyWnZRzVz\ncsKV2717N1OmTOHdd9/lyCOPZOjQobVXpAfe8Axz1V1Tpri4mJtvvpnt27eTm5vLQw89RO/evetc\n5rzzzuOLL76oDQBDhw7lq6++OugGcOL2p0yZwsSJE7n11luZOnUql19+eZ31q6/Oc+bMYeLEiXzz\nzTecfvrpzJ07N9R+3nzzzbz11lvk5uYyYMAArr32WpYvX15v+bqOYc10Xl4ef/zjH7n11lsZM2YM\nWVlZDBkypPYpqJbU4HgAZtYbeAXo4+67zey/gSXAGcCn7n6vmd0GdHb3yWbWl/iYwecAPYCXgXx3\ndzMrAm5w9yIzWwI8VNewkGbmizYsSlrxyg8qGZw7mAH9BqS4yyKZ01qfW5dDQ32/P40dDyBZF9AO\n4mMBtzezI4D2wIfASGBWUGYWcFmQvhR42t2r3b0MKAWGmNnxQI67FwXlZicsIyIiGdBgAAjG/r0P\n2Er8xP+5uy8Dct29KihWBdQ8AtAdKE9YRTnxlsCB+RVBvoiIZEiD9wDM7GTgJqA3sB1YYGbXJpYJ\nuneatU077+F5tel+g/vRb4ge9RQRqRGLxYjFYk1eT7KbwGcD/8/d/w5gZs8B/wxUmlk3d68Munc+\nDspXAImf0MsjfuVfEaQT8+u9xX31xKtT2gkRkSgpKCigoKCgdrqxH5dLdg9gA/BtM2tn8VvWI4AS\nYDEwLigzDlgYpBcBV5lZtpmdCOQDRe5eCewwsyHBesYmLCMiIhnQYAvA3dea2WygmPhjoH8FHgdy\ngPlmNp7gMdCgfImZzSceJPYAE3zfLesJxB8DbUf8MdCDngASEZH0afAx0EzQY6ByONNjoNIU6X4M\nVEREDlMKACICwJlnnsmrr76a6Wo06Cc/+Qm/+tWvMl2Nw0ar/BicSFTMfXYuO3e33JiQOW1zuObK\na0KVfeedd1qsHs1lxowZma7CYUUBQCSDdu7eSY+zW+6dyIrilv+gWFPs2bOHI47QaShT1AUkIkB8\ngPXEYSBHjRrFuHHj6NChA2eeeSarVq3ar+x9993HgAED6NSpE1dddRW7d++unf+nP/2JgQMH0rlz\nZ4YOHcq6dev2W/bee++lf//+5OTksHfvwZ91v/nmm8nNzaVjx47079+fkpISAH7wgx8wdepUIP4y\nVF5eHr/5zW847rjj6N69OwsXLmTJkiWceuqpdOnShbvvvrt2nUVFRZx99tl07NiRbt26ccsttzTv\nATwEKQCICHDw1y8XL17MmDFj2L59OyNHjuSGG27Yr+yCBQt46aWX+Nvf/sbbb79dO2bu6tWrGT9+\nPE888QTbtm3j+uuvZ+TIkVRXV9cu/8wzz/DnP/+Zzz///KDPQ7/00kusXLmSTZs2sX37dhYsWMAx\nxxxTu93EelZVVbF7924++ugj7rrrLn784x8zd+5cVq9ezcqVK7nrrrvYsmULADfeeGPtl0k3b97M\nqFGjmvX4HYoUAESkTsOGDaOwsBAz49prrz1oWMVJkybRrVs3OnfuzCWXXMKaNWsAePzxx7n++us5\n55xzMDO+//3v07ZtW9544w0gfhKfNGkSPXr0oG3btgdtNzs7m507d/Luu++yd+9eTjvttP1G+Ep8\nDPLII4/kF7/4BVlZWYwePZpt27Zx0003cfTRR9O3b1/69u1bW+/s7Gw2bdrEp59+Svv27RkyZEiz\nH7NDjQKAiNQpcZjH9u3bs2vXrv26axJPyu3atasdLnHLli3cd999dO7cufanvLycDz/8sLZ8z56J\nX4zZ3/nnn88NN9zAT3/6U3Jzc7n++utrh5g8UJcuXWpbBO3atTuo3on1qhlisk+fPgwePJgXXngh\n9LE4XCkAiEizqDkR9+rVi1/84hd89tlntT9ffPEFo0ePPqhsfSZOnEhxcTElJSVs3LiR3/zmN6GX\nrc8pp5zCvHnz+OSTT7jtttu48sor+eqrrxq1rsOFAoCINIuarpnrrruOxx57jKKiItydL7/8khde\neCH0gOrFxcW8+eabVFdX0759+/2Gp3T3Rr9JPWfOHD755BMAOnbs2ODwlFGh569EMiinbU6LPqqZ\n0zbkmJAHOPBma01emPKDBg3iiSee4IYbbmDTpk20a9eOYcOG7ff1yobs2LGDm2++mc2bN3PUUUdR\nWFjIz3/+8zrrlUodX3rpJW655Rb+8Y9/0Lt3b5555pk670FEib4FJJJG+haQNIW+BSQiIs1CAUBE\nJKIUAEREIkoBQEQkopIGADM7zcxWJ/xsN7NJZnaMmS0zs41mttTMOiUsM8XMNpnZBjO7ICF/kJmt\nC+Y92FI7JSIiySUNAO7+nruf5e5nAYOAfwDPA5OBZe5+KrA8mMbM+gKjgb5AIfCo7Xs2awYw3t3z\ngXwzK2zuHRIRkXBSfQ9gBFDq7h+Y2UhgeJA/C4gRDwKXAk+7ezVQZmalwBAz2wLkuHtRsMxs4DJA\nYwNLpDT2TVaR5pZqALgKeDpI57p7VZCuAmo+wNEdeCNhmXKgB1AdpGtUBPkikaF3AKQ1CR0AzCwb\nuAS47cB57u5m1my/2fMenleb7je4H/2G9GuuVYuIHPJisRixWKzJ60mlBXAhsMrdPwmmq8ysm7tX\nmtnxwMdBfgWQ+Km/POJX/hVBOjG/znfgr554dQrVEhGJloKCgv0+rTFt2rRGrSeVx0DHsK/7B2AR\nMC5IjwMWJuRfZWbZZnYikA8UuXslsMPMhgQ3hccmLCMiImkWqgVgZkcTvwF8XUL23cB8MxsPlAGj\nANy9xMzmAyXAHmCC7+v4nADMBNoBS9xdN4BFRDIkVABw9y+BrgfkbSMeFOoqPx2YXkf+KkAd+iIi\nrYDeBBYRiSgFABGRiFIAEBGJKAUAEZGIUgAQEYkoBQARkYhSABARiSgFABGRiFIAEBGJKAUAEZGI\nUgAQEYkoBQARkYhSABARiSgFABGRiFIAEBGJKAUAEZGIChUAzKyTmT1rZu+aWUkwrOMxZrbMzDaa\n2VIz65RQfoqZbTKzDWZ2QUL+IDNbF8x7sCV2SEREwgnbAniQ+BCOfYD+wAZgMrDM3U8FlgfTmFlf\nYDTQFygEHg3GAAaYAYx393wg38wKm21PREQkJUkDgJl1BIa5+1MA7r7H3bcDI4FZQbFZwGVB+lLg\naXevdvcyoBQYYmbHAznuXhSUm52wjIiIpFmYFsCJwCdm9jsz+6uZPREMEp/r7lVBmSogN0h3B8oT\nli8HetSRXxHki4hIBoQZFP4I4J+AG9z9LTN7gKC7p4a7u5l5c1Vq3sPzatP9Bvej3xCNIy8iUiMW\nixGLxZq8njABoBwod/e3gulngSlApZl1c/fKoHvn42B+BdAzYfm8YB0VQToxv6KuDV498erweyAi\nEjEFBQUUFBTUTk+bNq1R60naBeTulcAHZnZqkDUCWA8sBsYFeeOAhUF6EXCVmWWb2YlAPlAUrGdH\n8ASRAWMTlhERkTQL0wIAmAjMNbNs4H3gh0AWMN/MxgNlwCgAdy8xs/lACbAHmODuNd1DE4CZQDvi\nTxW92Ez7ISIiKQoVANx9LXBOHbNG1FN+OjC9jvxVgDr0RURaAb0JLCISUQoAIiIRpQAgIhJRCgAi\nIhGlACAiElEKACIiEaUAICISUQoAIiIRFfZN4LR67bW1Sct8/sk2cs/IZUC/AWmokYjI4adVBgDs\ntKRFvvhiA7t2V6ehMiIih6dWGQCyjzwqaZk2WVlpqImIyOFL9wBERCJKAUBEJKIUAEREIkoBQEQk\nokIFADMrM7O3zWy1mRUFeceY2TIz22hmS82sU0L5KWa2ycw2mNkFCfmDzGxdMO/B5t8dEREJK2wL\nwIECdz/L3QcHeZOBZe5+KrA8mMbM+gKjgb5AIfBoMAQkwAxgvLvnA/lmVthM+yEiIilKpQvIDpge\nCcwK0rOAy4L0pcDT7l7t7mVAKTAkGDg+x92LgnKzE5YREZE0S6UF8LKZFZvZdUFerrtXBekqIDdI\ndwfKE5YtB3rUkV8R5IuISAaEfRFsqLt/ZGbHAsvMbEPiTHd3M/N6lhURkVYo7KDwHwX/fmJmzwOD\ngSoz6+bulUH3zsdB8QqgZ8LiecSv/CuCdGJ+RV3bW/HcvNr0CX360buPxpEXEakRi8WIxWJNXo+5\nN3zhbmbtgSx332lmRwNLgWnACODv7n6PmU0GOrn75OAm8DziQaIH8DJwStBKeBOYBBQBLwAPufuL\nB2zP/332oqQVLytdxyX9TmPUlVekuMsiIocXM8PdD7xPm1SYFkAu8HzwIM8RwFx3X2pmxcB8MxsP\nlAGjANy9xMzmAyXAHmCC74syE4CZQDtgyYEnfxERSZ+kAcDd/wYMrCN/G/FWQF3LTAem15G/ClB/\njohIK6A3gUVEIkoBQEQkohQAREQiSgFARCSiFABERCJKAUBEJKIUAEREIkoBQEQkohQAREQiSgFA\nRCSiFABERCJKAUBEJKIUAEREIkoBQEQkohQAREQiSgFARCSiQgUAM8sys9VmtjiYPsbMlpnZRjNb\namadEspOMbNNZrbBzC5IyB9kZuuCeQ82/66IiEgqwrYAbiQ+xGPN0I6TgWXufiqwPJgmGA94NNAX\nKAQetWAsSWAGMN7d84F8Mytsnl0QEZHGSBoAzCwPuAj4L6DmZD4SmBWkZwGXBelLgafdvdrdy4BS\nYIiZHQ/kuHtRUG52wjIiIpIBYVoA/wf4ObA3IS/X3auCdBXxgeMBugPlCeXKgR515FcE+SIikiEN\nBgAzuxj42N1Xs+/qfz/u7uzrGhIRkUPEEUnmnwuMNLOLgKOADmb2e6DKzLq5e2XQvfNxUL4C6Jmw\nfB7xK/+KIJ2YX1HfRlc8N682fUKffvTu0y/k7oiIHP5isRixWKzJ67H4BXyIgmbDgZ+5+yVmdi/w\nd3e/x8wmA53cfXJwE3geMJh4F8/LwCnu7mb2JjAJKAJeAB5y9xfr2I7/++xFSetTVrqOS/qdxqgr\nrwi3pyIihykzw93r7KVpSLIWwIFqosXdwHwzGw+UAaMA3L3EzOYTf2JoDzDB90WYCcBMoB2wpK6T\nv4iIpE/oAODuK4AVQXobMKKectOB6XXkrwLUlyMi0kroTWARkYhSABARiSgFABGRiFIAEBGJKAUA\nEZGIUgAQEYkoBQARkYhSABARiSgFABGRiFIAEBGJKAUAEZGIUgAQEYkoBQARkYhSABARiSgFABGR\niFIAEBGJqGSDwh9lZm+a2RozKzGzXwf5x5jZMjPbaGZLzaxTwjJTzGyTmW0wswsS8geZ2bpg3oMt\nt0siIhJGgwHA3XcB57v7QKA/cL6Z/QswGVjm7qcCy4NpgjGBRwN9gULgUTOrGadyBjDe3fOBfDMr\nbIkdEhGRcJJ2Abn7P4JkNpAFfAaMBGYF+bOAy4L0pcDT7l7t7mVAKTDEzI4Hcty9KCg3O2EZERHJ\ngKQBwMzamNkaoAr4i7uvB3LdvSooUgXkBunuQHnC4uVAjzryK4J8ERHJkKSDwrv7XmCgmXUEXjKz\n8w+Y72bmzVmpFc/Nq02f0KcfvftoLHkRkRqxWIxYLNbk9SQNADXcfbuZvQAMAqrMrJu7VwbdOx8H\nxSqAngmL5RG/8q8I0on5FfVta/jlV4etlohI5BQUFFBQUFA7PW3atEatJ9lTQF1rnvAxs3bAd4HV\nwCJgXFBsHLAwSC8CrjKzbDM7EcgHity9EthhZkOCm8JjE5YREZEMSNYCOB6YZWZtiAeL37v7cjNb\nDcw3s/FAGTAKwN1LzGw+UALsASa4e0330ARgJtAOWOLuLzb3zoiISHgNBgB3Xwf8Ux3524AR9Swz\nHZheR/4qQJ35IiKthN4EFhGJKAUAEZGIUgAQEYkoBQARkYhSABARiSgFABGRiFIAEBGJKAUAEZGI\nUgAQEYkoBQARkYhSABARiSgFABGRiFIAEBGJKAUAEZGIUgAQEYkoBQARkYhKGgDMrKeZ/cXM1pvZ\nO2Y2Kcg/xsyWmdlGM1taM3RkMG+KmW0ysw1mdkFC/iAzWxfMe7BldklERMII0wKoBm529zOAbwM/\nNbM+wGRgmbufCiwPpjGzvsBooC9QCDwajAMMMAMY7+75QL6ZFTbr3oiISGhJA4C7V7r7miD9BfAu\n0AMYCcwKis0CLgvSlwJPu3u1u5cBpcAQMzseyHH3oqDc7IRlREQkzVK6B2BmvYGzgDeBXHevCmZV\nAblBujtQnrBYOfGAcWB+RZAvIiIZ0OCg8InM7FvAH4Ab3X3nvl4dcHc3M2+uSq14bl5t+oQ+/ejd\nR2PJi4jUiMVixGKxJq8nVAAwsyOJn/x/7+4Lg+wqM+vm7pVB987HQX4F0DNh8TziV/4VQToxv6Ku\n7Q2//OrweyAiEjEFBQUUFBTUTk+bNq1R6wnzFJABTwIl7v5AwqxFwLggPQ5YmJB/lZllm9mJQD5Q\n5O6VwA4zGxKsc2zCMiIikmZhWgBDgWuBt81sdZA3BbgbmG9m44EyYBSAu5eY2XygBNgDTHD3mu6h\nCcBMoB2wxN1fbKb9EBGRFCUNAO7+GvW3FEbUs8x0YHod+asAdeiLiLQCehNYRCSiFABERCJKAUBE\nJKIUAEREIkoBQEQkohQAREQiSgFARCSiFABERCJKAUBEJKIUAEREIkoBQEQkohQAREQiSgFARCSi\nFABERCJKAUBEJKIUAEREIirMkJBPmVmVma1LyDvGzJaZ2UYzW2pmnRLmTTGzTWa2wcwuSMgfZGbr\ngnkPNv+uiIhIKsK0AH4HFB6QNxlY5u6nAsuDacysLzAa6Bss82gw/i/ADGC8u+cD+WZ24DpFRCSN\nkgYAd18JfHZA9khgVpCeBVwWpC8Fnnb3ancvA0qBIWZ2PJDj7kVBudkJy4iISAY09h5ArrtXBekq\nIDdIdwfKE8qVAz3qyK8I8kVEJEOSDgqfjLu7mXlzVKbGiufm1aZP6NOP3n00jryISI1YLEYsFmvy\nehobAKrMrJu7VwbdOx8H+RVAz4RyecSv/CuCdGJ+RX0rH3751aEqsXLlX9n2aXboSufkwDXXXBK6\nvIhIpsx9di47d++sd363/G5N3kZjA8AiYBxwT/DvwoT8eWZ2P/EunnygKGgl7DCzIUARMBZ4qEk1\nB3btMnr0CH9Cr6hY3NRNioikxc7dO+lxdsv2lCcNAGb2NDAc6GpmHwC3A3cD881sPFAGjAJw9xIz\nmw+UAHuACe5e0z00AZgJtAOWuPuLzbsrIiKSiqQBwN3H1DNrRD3lpwPT68hfBagzX0SkldCbwCIi\nEaUAICISUQoAIiIRpQAgIhJRTX4R7FBSVLQqdFm9MyAih7tIBYBdu44I/d6A3hkQkcOduoBERCJK\nAUBEJKIUAEREIkoBQEQkoiJ1EzgVqTwxBHpqSESaV1HR2xz1ab0fTW4WCgD1SOWJIdBTQyLSvHbt\ngh7HDG7RbagLSEQkotQCaCZ6yUxEDjUKAM1EL5mJyKEm7QHAzAqBB4As4L/c/Z501yHTUr3B/Pbb\nq+jff1CosmpdiEhYaQ0AZpYFPEJ8MJkK4C0zW+Tu7zZmfaVb1vHn1x4LXb7sw7WN2UyzC9NaWLcu\nRr9+BQC8+uraFmtdzJ27mJ31Dzu6n0wFl1gsRkFBQdq32xrpWOxzqB2LVP7WAMrKyjn5X1quPpD+\nFsBgoNTdywDM7BngUqBRAaC6zdd0OTP8mJmVS98PHTAyHSwSA0AqUm1dFBWt5Xvf+/dQZZ9//s6U\nfoFTabk0VP5Pf5rHhg07Q5Wtz+HSMjrUTnot6VA7Fjt3ktKThdXVT7RgbeLSHQB6AB8kTJcDQ9K1\n8W+y9oQOGKkEC0gtYJR9uDbpuku3FteWSWXdqT6+umtXy607lZZLQ+VzclYdlJ/qulMJXodLsEjm\nUGj91aWm3sXF7/HYY8lbvKnUPdWr9FQuROIXW63jGNZIdwDw5EVg89rXk5bZ9eWXgDW1PvVKJVhA\nagGjcueLh/hTAAAENUlEQVT7dDnz+w2Wabcxp3b7qaz7rfWL6PRa11BlofkDV2PX3dD6E4NhY9ed\nSvD67cOX88Krfw697o/LK7nykh+GKptqy+XZxb/juLxutdPrit9i40cfN7ke0Hpaf6kek5p613Vh\nUJdU/j//WlzMd0b8KHRd1pSs58IL7wxVdsPm2Sn9/fx9e3noso1l7qHOyc2zMbNvA3e6e2EwPQXY\nm3gj2MzSVyERkcOEu6d8RZzuAHAE8B7wv4APgSJgTGNvAouISOOltQvI3feY2Q3AS8QfA31SJ38R\nkcxIawtARERaj4x8C8jMCs1sg5ltMrPb6inzUDB/rZmdle46pkuyY2Fm1wTH4G0ze93M+meinukQ\n5vciKHeOme0xs8vTWb90Cvk3UmBmq83sHTOLpbmKaRPib6Srmb1oZmuCY/GDDFSzxZnZU2ZWZWbr\nGiiT2nnT3dP6Q7zrpxToDRwJrAH6HFDmImBJkB4CvJHueraiY/HPQMcgXRjlY5FQ7hXgT8AVma53\nBn8vOgHrgbxgumum653BY3En8Oua4wD8HTgi03VvgWMxDDgLWFfP/JTPm5loAdS+DObu1UDNy2CJ\nRgKzANz9TaCTmeWmt5ppkfRYuPv/uPv2YPJNIC/NdUyXML8XABOBZ4FP0lm5NAtzLK4G/uDu5QDu\n/mma65guYY7FR0CHIN0B+Lu770ljHdPC3VcCnzVQJOXzZiYCQF0vgx34wH1dZQ7HE1+YY5FoPLCk\nRWuUOUmPhZn1IP7HPyPIOlxvYIX5vcgHjjGzv5hZsZmNTVvt0ivMsXgCOMPMPgTWAjemqW6tTcrn\nzUx8DTTsH+2Bz7Qejn/soffJzM4HfgQMbbnqZFSYY/EAMNnd3cyMlnwTMLPCHIsjgX8i/kh1e+B/\nzOwNd9/UojVLvzDH4n8Da9y9wMxOBpaZ2QB3T+HVtcNGSufNTASACqBnwnRP4pGqoTJ5Qd7hJsyx\nILjx+wRQ6O4NNQEPZWGOxSDgmfi5n67AhWZW7e6L0lPFtAlzLD4APnX3r4CvzOxVYABwuAWAMMfi\nXOA/ANz9fTP7G3AaUJyWGrYeKZ83M9EFVAzkm1lvM8sGRgMH/gEvAr4PtW8Pf+7uVemtZlokPRZm\n1gt4DrjW3UszUMd0SXos3P0kdz/R3U8kfh/gJ4fhyR/C/Y38EfgXM8sys/bEb/qVpLme6RDmWGwg\n/oVhgj7v04DNaa1l65DyeTPtLQCv52UwM7s+mP9/3X2JmV1kZqXAl0D4D5wcQsIcC+B2oDMwI7jy\nrXb3lh0oNANCHotICPk3ssHMXgTeBvYCT7j7YRcAQv5eTAd+Z2ZriV/U3uru2zJW6RZiZk8Dw4Gu\nZvYBcAfxrsBGnzf1IpiISERpUHgRkYhSABARiSgFABGRiFIAEBGJKAUAEZGIUgAQEYkoBQARkYhS\nABARiaj/D3xbHqr669gxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x118f20e48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "bins = np.linspace(0, 1, 30)\n",
    "alpha = 0.3\n",
    "\n",
    "plt.hist(sims, bins=bins, alpha=alpha, label='sims with shuffle')\n",
    "plt.hist(inner_sims, bins=bins, alpha=alpha, label='inner sims')\n",
    "plt.legend(loc='best');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.63160000000000005"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(sims > 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAEACAYAAAC3adEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAHm9JREFUeJzt3Xt0VeW97vHvQxBEGkRQo4SbOmKRFsVqm566kWXxOKhD\n0VaH4AU53enpGccWbUctFT1bqR1eq6LW264b5WIUtVZEy7ZyaEFkFKNuxWwpBawIRImKbIhWOYC/\n80dmwpoQyFoJufJ8xmBkzne+77veOUnms+ZlramIwMzMrE6Xth6AmZm1Lw4GMzNLcTCYmVmKg8HM\nzFIcDGZmluJgMDOzlJyCQVJvSb+T9FdJyyWVSuojab6klZJekNQ7q/5kSaskrZB0Rlb5SZIqk2V3\nZZV3l/R4Ur5U0qB9u5pmZparXI8Y7gLmRcRxwPHACuAqYH5EHAssSOaRNBQYCwwFRgP3SVLSz/1A\nWUSUACWSRiflZcDGpHwqcEuz18zMzJqk0WCQdDAwIiIeAoiI7RGxGRgDzEiqzQDOTabPAR6LiG0R\nsQZYDZRKOhIojIiKpN7MrDbZfT0FjGrWWpmZWZPlcsRwFPChpIcl/YekByX1BIoiojqpUw0UJdP9\ngPVZ7dcDxQ2UVyXlJD/XQW3wAJsl9WnKCpmZWfPkEgxdga8B90XE14BPSU4b1Yna79Xwd2uYmXUC\nXXOosx5YHxGvJPO/AyYDGyQdEREbktNEHyTLq4ABWe37J31UJdO7lte1GQi8J6krcHBEfJw9CEkO\nHjOzJogINV5rp0aPGCJiA7BO0rFJ0enAW8CzwISkbAIwJ5meC4yT1E3SUUAJUJH0syW5o0nAeOCZ\nrDZ1fZ1P7cXshsbifxFcd911bT6G9vLP28Lbwtti7/+aIpcjBoCJQLmkbsDbwPeBAuAJSWXAGuCC\nZOe9XNITwHJgO3BZ7BzdZcB0oAe1dzk9n5RPA2ZJWgVsBMY1aW3MzKzZcgqGiFgGfL2BRafvof6N\nwI0NlL8GDGugfCtJsJiZWdvyJ587oEwm09ZDaDe8LXbyttjJ26J51NRzUK1NUnSUsZqZtReSiDwv\nPud6jaHd2vmharOOwW9wrL3r8MEA/kOzjsNvZKwj8DUGMzNLcTCYmVmKg8HMzFIcDNZqFi9ezJAh\nQ/a4fM2aNXTp0oUvvviiWa+zcOFCBgwY0HjFPViyZAklJSUUFhYyd+5cqqurOfXUU+nVqxdXXnkl\nU6ZMYfz48c0ao1l71ikuPu+qvPxZamparv/CQrj44rNb7gXamUwmw/jx4ykrK2tWPyNGjGDFihX1\n84MHD+ahhx7i29/+dnOHuE9de+21XH755UycOBGAX/3qVxx++OFs2bIFgF/+8pdtOTyzFtcpg6Gm\nBoqLW27HXVX1bIv13ZDt27fTtWvb/Ve11J00yf3VLdJ3c6xdu5ahQ4fWz7/77rscd9xx9fPtccxm\n+5JPJbWgwYMHc/vtt3PCCSfQu3dvxo0bx9atW+uXP/jgg5SUlNC3b1/OOecc3n///fplXbp04b77\n7qOkpIQvf/nLLFq0iP79+/PrX/+aww8/nH79+jFnzhzmzZvHscceS9++fbn55pv3OJbNmzdz6aWX\ncvjhhzN48GBuuOGG+h3crqdG6k7p7Nixg2uuuYbFixfz4x//mMLCQi6//PLd+p4wYQJ33HEHAFVV\nVfVjB3j77bfp27cvkD7FM378eNauXcvZZ59NYWEht912W31/jzzyCIMGDeKwww7jxht3+2aVevPm\nzeMrX/kKvXr1on///tx+++2p5XfccQdFRUX069eP6dOn15dnMhmmTZtWPz99+nRGjBgBwDHHHMPf\n//73+nFddNFFzJw5k1tvvZVevXqxYMGC3YJy6dKlfOtb3+KQQw5h+PDhLFq0aI9jNusIHAwtSBJP\nPvkkf/zjH3nnnXd4880363dQf/rTn7j66qt58sknef/99xk0aBDjxqW/O/CZZ57hlVdeYfny5UQE\n1dXVbN26lffff5/rr7+eH/zgB5SXl/P666+zePFirr/+et59990GxzJx4kRqamp45513WLRoETNn\nzuThhx+uH+eexn/DDTcwYsQI7r33Xmpqarj77rt3q5fJZFi4cCEAixYt4uijj+bFF1+snz/11FN3\nazNr1iwGDhzIc889R01NDVdeeWX9siVLlrBy5UoWLFjA9ddfnzr9lK2srIzf/va3bNmyhbfeeit1\nSmrDhg1s2bKF9957j2nTpvGjH/2IzZs316/Xntb57bffTo3r0Ucf5eKLL+YXv/gFW7ZsYdSoUakj\nhqqqKs466yyuvfZaNm3axG233cZ5553HRx991GD/Zh2Bg6GFXX755RxxxBEccsghnH322bzxxhsA\nlJeXU1ZWxvDhw+nWrRs33XQTf/nLX1i7dm1928mTJ9O7d2+6d+8OwAEHHMA111xDQUEBY8eO5eOP\nP+YnP/kJPXv2ZOjQoQwdOrS+/2w7duzg8ccf56abbqJnz54MGjSIn/3sZ8yaNQvI7dTI3uqceuqp\nvPTSS0QEixcvZtKkSSxZsgSoDYaRI0fmvsGA6667ju7du3P88cdzwgknsGzZsgbrdevWjbfeeost\nW7Zw8MEHc+KJJ9YvO+CAA7j22mspKCjgO9/5Dl/60pf429/+ltc4su1p/R955BHOPPNMRo+ufXz5\n6aefzsknn8y8efOa/Fpmbc3B0MKOOOKI+ukePXrw6aefAtQfJdTp2bMnffv2paqqqr5s1ztr+vbt\nW/9Ot0ePHgAUFRXVL8/uP9tHH33Etm3bUq83cODA1Gs1Zm/XGY455hh69uzJG2+8weLFiznrrLPo\n168fK1eu5MUXX8w7GLK32UEHHdTgOgE89dRTzJs3j8GDB5PJZFi6dGn9sr59+9Kly85f74MOOohP\nPvkkr3Hk4t133+XJJ5/kkEMOqf+3ZMkSNmzYsM9fy6y1OBjaSL9+/VizZk39/KeffsrGjRspLi6u\nL9tXF30PPfRQDjjggNTrrV27lv79ax+o17NnT/7xj3/UL9t1p5bLOEaOHMmTTz7Jtm3b6NevHyNH\njmT69Ols2rSJ4cOHN9imuet38sknM2fOHD788EPOPfdcLrggt29u79mzZypsmrMTHzhwIOPHj2fT\npk31/2pqapg0aVKT+zRraw6GVlZ3SuLCCy/k4YcfZtmyZWzdupWrr76ab37zmwwcOHCfv2ZBQQEX\nXHAB11xzDZ988gnvvvsuU6dO5ZJLLgHgxBNP5MUXX2TdunVs3ryZm266KdW+qKiIt99+e6+vMXLk\nSO6555766wmZTIZ77rmHESNG7DEAcukXGj6Ns23bNsrLy9m8eTMFBQUUFhZSUFDQaF8Aw4cP5/e/\n/z2fffYZq1evTl2IzvX161xyySU8++yzvPDCC+zYsYPPP/+chQsX5nU0ZtbedMrbVQsLW/aW0sLC\nprXLvug5atQofvWrX3HeeeexadMmTjnlFGbPnp2q21D7vc3vzW9+8xsmTpzI0UcfzYEHHsgPf/hD\nvv/97wO158XHjh3L8ccfz2GHHcakSZN47rnn6tteccUVTJgwgfvvv59LL72UO++8c7f+Tz31VD75\n5JP6YDjllFP47LPPdrvwnD3myZMnM3HiRCZNmsS//Mu/8L3vfS+n9a7zyCOPMHHiRHbs2MGQIUMo\nLy/Padv89Kc/5ZVXXqGoqIgTTjiBSy65hAULGnyabH1f2f1lz/fv359nnnmGSZMmceGFF1JQUEBp\naWn9XVlmHVGHfx5De70X3qwh/n211taU5zH4VJKZmaU4GMzMLMXBYGZmKQ4GMzNLcTCYmVmKg8HM\nzFIcDGZmluJgMDOzFAeDmZmldMqvxCj/XTk1W1vu2Z6F3Qu5+PyLW6z/9mZfPdrTzDqGnIJB0hpg\nC7AD2BYR35DUB3gcGASsAS6IiP9K6k8G/jmpf3lEvJCUnwRMBw4E5kXEFUl5d2Am8DVgIzA2Ihp+\n4kwOarbWUHxyceMVm6jq1db9grTO+mhPM2ufcj2VFEAmIk6MiG8kZVcB8yPiWGBBMo+kocBYYCgw\nGrhPO/cs9wNlEVEClEganZSXARuT8qnALc1cr3Zhf3m0p5l1LvlcY9j1beMYYEYyPQM4N5k+B3gs\nIrZFxBpgNVAq6UigMCIqknozs9pk9/UUMCqPcbVb+8ujPc2sc8nniOH/SnpV0v9MyooiojqZrgbq\nHiXWD1if1XY9UNxAeVVSTvJzHUBEbAc2J6eqOrz94dGeZta55BoMp0TEicB3gB9JGpG9MPk+bO85\nGrA/PNrTzDqXnK5oRsT7yc8PJT0NfAOolnRERGxIThN9kFSvArL3aP2pPVKoSqZ3La9rMxB4T1JX\n4OCI+HjXcUyZMqV+OpPJkMlkchl+u9RWj/Y87rjjgH3/aE8zax2N3XW5cvlKVv51ZbNeo9FgkHQQ\nUBARNZJ6AmcAvwTmAhOovVA8AZiTNJkLPCrpDmpPEZUAFRERkrZIKgUqgPHA3VltJgBLgfOpvZi9\nm+xg6KiyH+154YUXctFFFzFkyJBWe7TnzJkz2bhxI1OnTuXnP/85UPtoz1tvvZV169bRq1evJj3a\n08xaR2N3XRafXMxpnFY//4en/5D3a+RyxFAEPJ28a+wKlEfEC5JeBZ6QVEZyuypARCyX9ASwHNgO\nXJb16LXLqL1dtQe1t6s+n5RPA2ZJWkXt7arpq7B5Kuxe2KK3lBZ2b9qzPTvzoz3NrPPwoz3NWpF/\nX625Hih/IK/PaY0ZMsaP9jQzs+ZxMJiZWYqDwczMUhwMZmaW4mAwM7MUB4OZmaV0iucx+JO5Zmb7\nTocPBt8Tbma2b/lUkpmZpTgYzMwsxcFgZmYpDgYzM0txMJiZWYqDwczMUhwMZmaW4mAwM7MUB4OZ\nmaU4GMzMLMXBYGZmKQ4GMzNLcTCYmVmKg8HMzFIcDGZmluJgMDOzFAeDmZmlOBjMzCzFwWBmZikO\nBjMzS3EwmJlZSk7BIKlA0uuSnk3m+0iaL2mlpBck9c6qO1nSKkkrJJ2RVX6SpMpk2V1Z5d0lPZ6U\nL5U0aF+uoJmZ5SfXI4YrgOVAJPNXAfMj4lhgQTKPpKHAWGAoMBq4T5KSNvcDZRFRApRIGp2UlwEb\nk/KpwC3NWyUzM2uORoNBUn/gTODfgLqd/BhgRjI9Azg3mT4HeCwitkXEGmA1UCrpSKAwIiqSejOz\n2mT39RQwqslrY2ZmzZbLEcNU4OfAF1llRRFRnUxXA0XJdD9gfVa99UBxA+VVSTnJz3UAEbEd2Cyp\nTx7rYGZm+1DXvS2UdBbwQUS8LinTUJ2ICEnR0LJ9bcqUKfXTmUyGTKbBIZmZ7bcqX66ksqKyWX3s\nNRiAbwFjJJ0JHAj0kjQLqJZ0RERsSE4TfZDUrwIGZLXvT+2RQlUyvWt5XZuBwHuSugIHR8THDQ0m\nOxjMzGx3w0qHMax0WP387Htn593HXk8lRcTVETEgIo4CxgF/iojxwFxgQlJtAjAnmZ4LjJPUTdJR\nQAlQEREbgC2SSpOL0eOBZ7La1PV1PrUXs83MrI00dsSwq7pTRjcDT0gqA9YAFwBExHJJT1B7B9N2\n4LKIqGtzGTAd6AHMi4jnk/JpwCxJq4CN1AaQmZm1Ee3cb7dvkqKjjNXMrKU8UP4AxScXN14xMWbI\nGCJCjdfcyZ98NjOzFAeDmZmlOBjMzCzFwWBmZikOBjMzS3EwmJlZioPBzMxSHAxmZpbiYDAzsxQH\ng5mZpTgYzMwsxcFgZmYpDgYzM0txMJiZWYqDwczMUhwMZmaW4mAwM7MUB4OZmaU4GMzMLMXBYGZm\nKQ4GMzNLcTCYmVmKg8HMzFIcDGZmluJgMDOzFAeDmZmlOBjMzCzFwWBmZil7DQZJB0p6WdIbkpZL\nuikp7yNpvqSVkl6Q1DurzWRJqyStkHRGVvlJkiqTZXdllXeX9HhSvlTSoJZYUTMzy81egyEiPgdO\ni4jhwPHAaZL+CbgKmB8RxwILknkkDQXGAkOB0cB9kpR0dz9QFhElQImk0Ul5GbAxKZ8K3LIvV9DM\nzPLT6KmkiPhHMtkNKAA2AWOAGUn5DODcZPoc4LGI2BYRa4DVQKmkI4HCiKhI6s3MapPd11PAqCav\njZmZNVujwSCpi6Q3gGrgzxHxFlAUEdVJlWqgKJnuB6zPar4eKG6gvCopJ/m5DiAitgObJfVp2uqY\nmVlzdW2sQkR8AQyXdDDwR0mn7bI8JEVLDTDblClT6qczmQyZTKY1XtbMrMOofLmSyorKZvXRaDDU\niYjNkv4AnARUSzoiIjYkp4k+SKpVAQOymvWn9kihKpnetbyuzUDgPUldgYMj4uOGxpAdDGZmtrth\npcMYVjqsfn72vbPz7qOxu5IOrbvjSFIP4L8DrwNzgQlJtQnAnGR6LjBOUjdJRwElQEVEbAC2SCpN\nLkaPB57JalPX1/nUXsw2M7M20tgRw5HADEldqA2RWRGxQNLrwBOSyoA1wAUAEbFc0hPAcmA7cFlE\n1J1mugyYDvQA5kXE80n5NGCWpFXARmDcvlo5MzPLn3but9s3SdFRxmpm1lIeKH+A4pOLG6+YGDNk\nDBGhxmvu5E8+m5lZioPBzMxSHAxmZpbiYDAzsxQHg5mZpTgYzMwsxcFgZmYpDgYzM0txMJiZWYqD\nwczMUhwMZmaW4mAwM7MUB4OZmaU4GMzMLMXBYGZmKQ4GMzNLcTCYmVmKg8HMzFIcDGZmluJgMDOz\nFAeDmZmlOBjMzCzFwWBmZikOBjMzS3EwmJlZioPBzMxSHAxmZpbiYDAzs5RGg0HSAEl/lvSWpP+U\ndHlS3kfSfEkrJb0gqXdWm8mSVklaIemMrPKTJFUmy+7KKu8u6fGkfKmkQft6Rc3MLDe5HDFsA34a\nEV8Bvgn8SNJxwFXA/Ig4FliQzCNpKDAWGAqMBu6TpKSv+4GyiCgBSiSNTsrLgI1J+VTgln2ydmZm\nlrdGgyEiNkTEG8n0J8BfgWJgDDAjqTYDODeZPgd4LCK2RcQaYDVQKulIoDAiKpJ6M7PaZPf1FDCq\nOStlZmZNl9c1BkmDgROBl4GiiKhOFlUDRcl0P2B9VrP11AbJruVVSTnJz3UAEbEd2CypTz5jMzOz\nfaNrrhUlfYnad/NXRETNzrNDEBEhKVpgfClTpkypn85kMmQymZZ+STOzDqXy5UoqKyqb1UdOwSDp\nAGpDYVZEzEmKqyUdEREbktNEHyTlVcCArOb9qT1SqEqmdy2vazMQeE9SV+DgiPh413FkB4OZme1u\nWOkwhpUOq5+ffe/svPvI5a4kAdOA5RFxZ9aiucCEZHoCMCerfJykbpKOAkqAiojYAGyRVJr0OR54\npoG+zqf2YraZmbWBXI4YTgEuAd6U9HpSNhm4GXhCUhmwBrgAICKWS3oCWA5sBy6LiLrTTJcB04Ee\nwLyIeD4pnwbMkrQK2AiMa+Z6mZlZEzUaDBHxEns+sjh9D21uBG5soPw1YFgD5VtJgsXMzNqWP/ls\nZmYpDgYzM0txMJiZWYqDwczMUhwMZmaW4mAwM7MUB4OZmaU4GMzMLMXBYGZmKQ4GMzNLcTCYmVmK\ng8HMzFIcDGZmluJgMDOzFAeDmZmlOBjMzCzFwWBmZikOBjMzS3EwmJlZioPBzMxSHAxmZpbiYDAz\nsxQHg5mZpTgYzMwsxcFgZmYpDgYzM0txMJiZWYqDwczMUhoNBkkPSaqWVJlV1kfSfEkrJb0gqXfW\nssmSVklaIemMrPKTJFUmy+7KKu8u6fGkfKmkQftyBc3MLD+5HDE8DIzepewqYH5EHAssSOaRNBQY\nCwxN2twnSUmb+4GyiCgBSiTV9VkGbEzKpwK3NGN9zMysmRoNhohYDGzapXgMMCOZngGcm0yfAzwW\nEdsiYg2wGiiVdCRQGBEVSb2ZWW2y+3oKGNWE9TAzs32kqdcYiiKiOpmuBoqS6X7A+qx664HiBsqr\nknKSn+sAImI7sFlSnyaOy8zMmqnZF58jIoDYB2MxM7N2oGsT21VLOiIiNiSniT5IyquAAVn1+lN7\npFCVTO9aXtdmIPCepK7AwRHxcUMvOmXKlPrpTCZDJpNp4vDNzDqnypcrqayobLziXjQ1GOYCE6i9\nUDwBmJNV/qikO6g9RVQCVERESNoiqRSoAMYDd+/S11LgfGovZjcoOxjMzGx3w0qHMax0WP387Htn\n591Ho8Eg6TFgJHCopHXAtcDNwBOSyoA1wAUAEbFc0hPAcmA7cFlyqgngMmA60AOYFxHPJ+XTgFmS\nVgEbgXF5r4WZme0zjQZDRFy4h0Wn76H+jcCNDZS/BgxroHwrSbCYmVnb8yefzcwsxcFgZmYpDgYz\nM0txMJiZWYqDwczMUhwMZmaW4mAwM7MUB4OZmaU4GMzMLMXBYGZmKQ4GMzNLcTCYmVlKU79228z2\nsfLyZ6mpya1uYSFcfPHZLTsg2285GMzaiZoaKC7ObWf/9NNTcg4RcJBYfhwMZi0knyMAgIqKZXz3\nu7ntvD//vGvOIQL5BYlDxBwMZnnIZ2dfu6P/Pzn3/fnny5o4qlz6zj1IqqqebbFxWMfgYDDLQz6n\ne1pyR9+SKipey6u+jzA6HweDmaXke5rKRxidj4PBzJolnyMMH110DA4GM2sWX7/ofBwMtl9ryTuH\nzDoqB4Pt1/K5mAwd94Jye+EL2x2Dg8HMWo0/f9ExOBjMrN3KJ0jy/TT4m2++xvHHn5Rz/ZYMnvLf\nlVOzNbfBV/xHBd89+bstMo46DgYz6xTyPRp58cVl7ea23JqtNRSfXJxT3c8rPm+xcdRxMJiZ5SCf\n6yP5Ho1U/PVNvptjMLQGB4N1Kr7LyFpKPkck+R6NrHj6X/n35ytyqrtmTVXO/TaVg8E6Fd9lZB3R\ntm1d6NvnGznV3bDh6ZxDpKnaTTBIGg3cCRQA/xYRt7TxkMzMmmTNe8v495ceyLn+xs3rc667Y0fu\nIdJU7SIYJBUA9wCnA1XAK5LmRsRf23Zk7dPChQvJZDJtPYx2wdtip8rKhQwblmnrYbQLLbEt8tnZ\nb6h5m75fvTTnvncs2N7UYbWIdhEMwDeA1RGxBkDSbOAcwMHQgP1tZ7i36wbPPfcoK1bsXLg/XzNw\nMOyUy7bI9119Pjv79rajz1d7CYZiYF3W/HqgtI3GYu3M3q4bFBa+llrmawbNl+8O85W35tL7pUP3\ned3m9L167auNrkNHf1ffktpLMEQulR5++OGcOvv617/OV7/61WYNyFrO/nLnUGvvYPe2M8yn73x3\nmP9vwef0/Wput1rmU7c5ffdYWdhou/1pR58vReS0T27ZQUjfBKZExOhkfjLwRfYFaEltP1Azsw4o\nIpRP/fYSDF2BvwGjgPeACuBCX3w2M2t97eJUUkRsl/Rj4I/U3q46zaFgZtY22sURg5mZtR9d2noA\nu5I0WtIKSask/WIPde5Oli+TdGJrj7G1NLYtJF2cbIM3JS2RdHxbjLOl5fI7kdT7uqTtkr7XmuNr\nTTn+fWQkvS7pPyUtbOUhtpoc/j4OlfS8pDeSbfE/2mCYrULSQ5KqJVXupU7u+82IaDf/qD2NtBoY\nDBwAvAEct0udM4F5yXQpsLStx92G2+K/AQcn06M747bIZTtk1fsT8BxwXluPuw1/J3oDbwH9k/lD\n23rcbbgtpgA31W0HYCPQta3H3kLbYwRwIlC5h+V57Tfb2xFD/QfdImIbUPdBt2xjgBkAEfEy0FtS\nUesOs1U0ui0i4i8RsTmZfRno38pjbA25/E4ATAR+B3zYmoNrZblsi4uApyJiPUBEfNTKY2wtuWyL\n94FeyXQvYGNEdMp7VCNiMbBpL1Xy2m+2t2Bo6INuu96M3FCdzrhDzGVbZCsD5rXoiNpGo9tBUjG1\nO4X7k6LOeuEsl9+JEqCPpD9LelXS+FYbXevKZVs8CHxF0nvAMuCKVhpbe5TXfrNd3JWUJdc/6F3v\nye2MO4Kc10nSacA/A6e03HDaTC7b4U7gqogISWL334/OIpdtcQDwNWpv/T4I+IukpRGxqkVH1vpy\n2RZXA29EREbSMcB8SSdERB4fr+xUct5vtrdgqAIGZM0PoDbZ9lanf1LW2eSyLUguOD8IjI6IvR1K\ndlS5bIeTgNm1mcChwHckbYuIua0zxFaTy7ZYB3wUEZ8Bn0l6ETgB6GzBkMu2+BZwA0BEvC3pHeDL\nwKutMsL2Ja/9Zns7lfQqUCJpsKRuwFhg1z/uucClUP+J6f+KiOrWHWaraHRbSBoI/B64JCJWt8EY\nW0Oj2yEijo6IoyLiKGqvM/zvThgKkNvfxzPAP0kqkHQQtRcal7fyOFtDLttiBbXf2ExyPv3LwN9b\ndZTtR177zXZ1xBB7+KCbpP+VLP/XiJgn6UxJq4FPge+34ZBbTC7bArgWOAS4P3m3vC0iWvaL2ltZ\njtthv5Dj38cKSc8DbwJfAA9GRKcLhhx/L24EHpa0jNo3wZMi4uM2G3QLkvQYMBI4VNI64DpqTys2\nab/pD7iZmVlKezuVZGZmbczBYGZmKQ4GMzNLcTCYmVmKg8HMzFIcDGZmluJgMDOzFAeDmZml/H+f\n3vXdYhG43QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10b14ae80>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "bins = np.linspace(0, 1, 30)\n",
    "alpha = 0.3\n",
    "\n",
    "plt.hist(np.linalg.norm(out_shuffled_activations, axis=1), bins=bins, alpha=alpha, label='norm out with shuffle')\n",
    "plt.hist(np.linalg.norm(out, axis=1), bins=bins, alpha=alpha, label='norm out')\n",
    "plt.legend(loc='best');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(59000,)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.linalg.norm(out, axis=1).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x, y = tt.dmatrix('x'), tt.dmatrix('y')\n",
    "d = tt.dot(x, y.T)\n",
    "compute_dot = theano.function([x, y], d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  14.,   38.,   62.],\n",
       "       [  38.,  126.,  214.],\n",
       "       [  62.,  214.,  366.]])"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compute_dot(np.arange(12).reshape(3, 4) * 1., np.arange(12).reshape(3, 4) * 1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
